diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml new file mode 100644 index 0000000..c777f4b --- /dev/null +++ b/.github/workflows/conventional-prs.yml @@ -0,0 +1,16 @@ +name: PR +on: + pull_request_target: + types: + - opened + - reopened + - edited + - synchronize + +jobs: + title-format: + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v3.4.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..4766b98 --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,50 @@ +on: + push: + branches: + - main + +name: release-please + +jobs: + release-please: + runs-on: ubuntu-latest + outputs: + release_created: ${{ steps.release.outputs.release_created }} + steps: + - uses: GoogleCloudPlatform/release-please-action@v3 + id: release + with: + release-type: python + package-name: snakemake-interface-executor-plugins + + publish: + runs-on: ubuntu-latest + needs: release-please + if: ${{ needs.release-please.outputs.release_created }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v2 + with: + python-version: "3.11" + + - name: Install poetry + run: pip install poetry + + - name: Determine dependencies + run: poetry lock + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + cache: poetry + + - name: Install Dependencies using Poetry + run: | + poetry install + + - name: Publish to PyPi + env: + PYPI_USERNAME: __token__ + PYPI_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: poetry publish --build --username $PYPI_USERNAME --password $PYPI_PASSWORD diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..2057e74 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,94 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches-ignore: [] + +jobs: + formatting: + runs-on: ubuntu-latest + steps: + - name: Check out the code + uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install poetry + run: pip install poetry + + - name: Determine dependencies + run: poetry lock + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + cache: poetry + + - name: Install Dependencies using Poetry + run: poetry install + + - name: Check formatting + run: poetry run black --check . + + linting: + runs-on: ubuntu-latest + steps: + - name: Check out the code + uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install poetry + run: pip install poetry + + - name: Determine dependencies + run: poetry lock + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + cache: poetry + + - name: Install Dependencies using Poetry + run: poetry install + + - name: Check code + run: poetry run flake8 + + testing: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install poetry + run: pip install poetry + + - name: Determine dependencies + run: poetry lock + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + cache: poetry + + - name: Install dependencies + run: | + pip install connection-pool # because it is incompatible with poetry + poetry install + + - name: Run pytest + run: poetry run coverage run -m pytest tests/tests.py + + - name: Run Coverage + run: poetry run coverage report -m diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bcc3d59 --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +poetry.lock \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4d4a950 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,341 @@ +# Changelog + +## [9.3.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.2.0...v9.3.0) (2024-10-04) + + +### Features + +* load builtin snakemake executor plugins ([#73](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/73)) ([03ee96b](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/03ee96be5047d68f7d9de951ab75458e7c79d3e3)) + +## [9.2.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.1.1...v9.2.0) (2024-07-04) + + +### Features + +* add a default for the `can_transfer_local_files` interface ([#67](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/67)) ([793df28](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/793df28ba733eb462fba7824f46729af65a58dc4)) +* support for commas in wildcards ([#56](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/56)) ([0e8ed82](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0e8ed82c2dc8338b402e646dde7ca48e02075922)) + +## [9.1.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.1.0...v9.1.1) (2024-04-12) + + +### Bug Fixes + +* pass cores to remote jobs if they are set ([395af5e](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/395af5e05c8b09d107415159819d0e3cef58717f)) + +## [9.1.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.0.2...v9.1.0) (2024-03-26) + + +### Features + +* add utils for encoding CLI args as base64 ([#64](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/64)) ([38a53ec](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/38a53ecec3af3fc45d2f962972460fa50258b2b1)) + +## [9.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.0.1...v9.0.2) (2024-03-22) + + +### Bug Fixes + +* quote list of args ([#62](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/62)) ([656ba0a](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/656ba0afb867301ccb48b24837fda1793e3281dc)) + +## [9.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.0.0...v9.0.1) (2024-03-21) + + +### Bug Fixes + +* fix quoting of string arguments that are passed to spawned jobs ([#60](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/60)) ([d3d55a3](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/d3d55a32dbd78be679727c3f95cd42308a9597ab)) + +## [9.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.2.0...v9.0.0) (2024-03-11) + + +### ⚠ BREAKING CHANGES + +* pass common settings to SpawedJobArgsFactory; shell command arg quoting fixes ([#58](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/58)) + +### Features + +* pass common settings to SpawedJobArgsFactory; shell command arg quoting fixes ([#58](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/58)) ([867a027](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/867a027e8abbfa8937900b648aeade91b01c2c38)) + +## [8.2.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.1.3...v8.2.0) (2024-01-16) + + +### Features + +* add ability to pass group args to remote jobs ([bcfd819](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/bcfd81953b3feeaac6669a3487cc1eab3d5a2727)) + +## [8.1.3](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.1.2...v8.1.3) (2023-12-19) + + +### Bug Fixes + +* break circular import ([aed33aa](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/aed33aa2aba20e229398deb5ad486d3b0ec7e213)) + +## [8.1.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.1.1...v8.1.2) (2023-12-12) + + +### Documentation + +* CommonSettings ([#50](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/50)) ([85b995d](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/85b995d726cd941ea0f6e43b6217e95140a82327)) + +## [8.1.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.1.0...v8.1.1) (2023-12-08) + + +### Bug Fixes + +* allow value of none for shared fs usage setting ([d334869](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/d33486933f41f2bccd099e2cab90b2d1a854def2)) + +## [8.1.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.0.2...v8.1.0) (2023-11-30) + + +### Features + +* add method for checking whether there is a common workdir assumed in storage settings ([29dc8dd](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/29dc8dd43ba4bd8d33eeda14a3dff9272d3751f0)) + + +### Bug Fixes + +* adapt to API changes ([21cae32](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/21cae32a8a69b58b732b773a849abfb02b533575)) + +## [8.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.0.1...v8.0.2) (2023-11-20) + + +### Bug Fixes + +* fix arg passing ([caee462](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/caee46241f4fc639ed585761421d335f7783399c)) + +## [8.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.0.0...v8.0.1) (2023-11-20) + + +### Bug Fixes + +* cleanup ci ([061ff4c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/061ff4ce7a6ef699dfff58c149195425bef13e86)) +* fix method name ([16138ad](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/16138ad9e085a289590b8308cc954662e8df0ffe)) + +## [8.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v7.0.3...v8.0.0) (2023-11-20) + + +### ⚠ BREAKING CHANGES + +* added common setting for defining whether workflow sources shall be deployed. + +### Features + +* added common setting for defining whether workflow sources shall be deployed. ([04319bb](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/04319bbe410275eea28cbe47d2abfe9b0b50c3e5)) + +## [7.0.3](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v7.0.2...v7.0.3) (2023-10-26) + + +### Bug Fixes + +* fix envvar declarations code ([fc31775](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/fc31775075f8b6ac6317b3762bcd385f31a8b746)) +* improved precommand handling ([af1f010](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/af1f01006fd5e7a493659e0bcb80e570628e5176)) + +## [7.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v7.0.1...v7.0.2) (2023-10-20) + + +### Bug Fixes + +* ignore errors when trying to delete tmpdir upon shutdown ([#39](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/39)) ([406422c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/406422c967ebd33227c34e257b0a1a5cdd0a3e4d)) + +## [7.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v7.0.0...v7.0.1) (2023-10-17) + + +### Miscellaneous Chores + +* release 7.0.1 ([c51e47b](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/c51e47b5b3eed7a5d52e27dead1d51659563aa9d)) + +## [7.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v6.0.0...v7.0.0) (2023-10-17) + + +### ⚠ BREAKING CHANGES + +* move behavior args into common settings and use __post_init__ method for additional initialization + +### Features + +* move behavior args into common settings and use __post_init__ method for additional initialization ([c6cb3c9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/c6cb3c9c02de7e7aeba241558ba549a65abcfc2b)) +* support precommand ([32da209](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/32da20943b1afe8854566356ed448015e4f67e6c)) + +## [6.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v5.0.2...v6.0.0) (2023-10-12) + + +### ⚠ BREAKING CHANGES + +* adapt to API changes + +### Features + +* adapt to API changes ([f74151b](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/f74151b32a9b98a323bbbf88a818b7da5fe97427)) + + +### Bug Fixes + +* adapt to changes in snakemake ([e572f02](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/e572f02c7b5270fc02fc871ac6197575ce42ad5c)) +* cleanup interfaces ([88c6554](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/88c65546db32fc5e48827173bb016d69691c41cb)) +* udpate deps ([99b5b1e](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/99b5b1e61302d75fb6ca7a959fda18cceaf12703)) + +## [5.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v5.0.1...v5.0.2) (2023-09-22) + + +### Documentation + +* mention poetry plugin ([733f2f9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/733f2f93b0e1fedb9aeda21ea6987b7b7059be11)) + +## [5.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v5.0.0...v5.0.1) (2023-09-22) + + +### Bug Fixes + +* adapt to changes in snakemake-interface-common ([faa05a4](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/faa05a40068e656e533671324c4a3928158e652e)) +* adapt to fixes in snakemake-interface-common ([2a92560](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/2a92560fa602cab4b3085643324bdaaa36d1ea42)) + +## [5.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v4.0.1...v5.0.0) (2023-09-21) + + +### ⚠ BREAKING CHANGES + +* maintain Python 3.7 compatibility by moving settings base classes to the settings module + +### Bug Fixes + +* maintain Python 3.7 compatibility by moving settings base classes to the settings module ([71c976e](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/71c976ea2a51afa418683effd3db9d80dca15150)) +* use bugfix release of snakemake-interface-common ([2441fc3](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/2441fc36fc0cfc404aafeb0d8b86e7f107c7ebb6)) + +## [4.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v4.0.0...v4.0.1) (2023-09-20) + + +### Bug Fixes + +* return correct value for next_seconds_between_status_checks ([0606922](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/06069228debfc55629f2eb6f2e88ac1b81ad90c8)) + +## [4.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v3.0.2...v4.0.0) (2023-09-19) + + +### ⚠ BREAKING CHANGES + +* rename ExecutorJobInterface into JobExecutorInterface + +### Code Refactoring + +* rename ExecutorJobInterface into JobExecutorInterface ([9f61b6a](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/9f61b6a5f16ab39582429b813640fa08f3e0231c)) + +## [3.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v3.0.1...v3.0.2) (2023-09-12) + + +### Bug Fixes + +* add error details in case of improper join_cli_args usage ([cb0245f](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/cb0245fe47adfc73e07600821b5813687025ad9c)) + +## [3.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v3.0.0...v3.0.1) (2023-09-11) + + +### Bug Fixes + +* avoid dependeing on argparse_dataclass fork ([0a1f02d](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0a1f02d5facf81a48ab687d8cb2809aebd6518d8)) +* fix NoneType definition ([1654a41](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/1654a4140b7ee91c5e7f7370795fd67e5e70014b)) + +## [3.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v2.0.0...v3.0.0) (2023-09-11) + + +### ⚠ BREAKING CHANGES + +* unify self.report_job_error and self.print_job_error. + +### Features + +* add further metadata to ExecutorSettings ([30f0977](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/30f0977a646f13bc86a649e3e76ddfbf417f3ace)) +* add get_items_by_category method to ExecutorSettings ([7f62bb9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/7f62bb9d15aa80f87964974d7a0bca504990e540)) +* add support for env_var specification in ExecutorSettings ([a1e3123](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/a1e3123a80db7b96bdb3cca11fa3faa21ab90ab3)) +* unify self.report_job_error and self.print_job_error. ([2f24fb9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/2f24fb938cef05abf912e4a66a066fdce414f06b)) + + +### Documentation + +* update readme ([100bdc0](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/100bdc015ef2e8af4aa35fb2a027f46aeb73d244)) +* update readme ([836b893](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/836b893287c8abed89dc738f9a3f48c335d9827a)) + +## [2.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.2.0...v2.0.0) (2023-09-08) + + +### ⚠ BREAKING CHANGES + +* rename ExecutorPluginRegistry.get to get_plugin. +* naming +* improved API + +### Features + +* add touch_exec ([0ac8b16](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0ac8b16e86419267e6cea49dee3451ed22fbde80)) +* allow to set the next sleep time ([f8fde6c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/f8fde6c0cbdbaf7cf164db65aae3ead5f5db919a)) +* allow to specify the initial amount of seconds to sleep before checking job status ([0e88e6f](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0e88e6ff4d3547c2c0991bc9d172413c9ed0d70b)) +* improved API ([0226c9d](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0226c9d2e7ab330f8552827f9714a43ff7f805c5)) +* naming ([978f74c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/978f74cab5ab1e829f1636993dd46b8b51589ce8)) +* rename ExecutorPluginRegistry.get to get_plugin. ([c1b50d9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/c1b50d9fb433d685749c10319827dea973caa8b2)) +* simplify API ([3e4be2a](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/3e4be2af66fd6fdbd3b0b60562023a4bdc64f92e)) + + +### Bug Fixes + +* API typing ([c9180fa](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/c9180fa55897e1b974edb068531f4cd6edce8d15)) + +## [1.2.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.1.2...v1.2.0) (2023-09-05) + + +### Features + +* simplify executor API ([7479c1c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/7479c1c98f0fdf04ee77cea11feae4da2421ff90)) +* various API improvements ([a2808ad](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/a2808ad1ec480949e88efc442532952f42d8f450)) + +## [1.1.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.1.1...v1.1.2) (2023-09-01) + + +### Bug Fixes + +* convert enum to cli choice ([a2f287c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/a2f287c9cd66d4b4ccb847434ee9294c4749b233)) +* various adaptations to changes in Snakemake 8.0 ([58ff504](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/58ff50488b49b1a34e41c4fa9812297430cc9672)) + +## [1.1.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.1.0...v1.1.1) (2023-08-30) + + +### Bug Fixes + +* practical improvements ([f91133a](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/f91133af160aef941e3663cfe3a50653589244f3)) + +## [1.1.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.0.1...v1.1.0) (2023-08-28) + + +### Features + +* refactor and clean up interfaces ([#14](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/14)) ([fc28032](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/fc28032030204504e26c148e73ef8d85af9a5cf7)) + +## [1.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.0.0...v1.0.1) (2023-08-02) + + +### Bug Fixes + +* release process fix ([b539c1b](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/b539c1b6795cfff9440bbd7e283e51c2df6518ba)) + +## 1.0.0 (2023-08-02) + + +### Features + +* migrate interfaces from snakemake to this package ([#7](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/7)) ([cc3327c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/cc3327c1e3020ff25f72f93f4c2711d7cadb11e6)) +* migrate snakemake.common.Mode into this package ([b0aa928](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/b0aa928d30f4cf49d459b8fa6ed6904d269f9d27)) +* object oriented plugin interface implementation ([a6923d2](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/a6923d2e5319124b5db2b72210280c43c5a47624)) +* start of work to integrate functions ([#5](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/5)) ([56f16d8](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/56f16d8f8ce9b0bf47d7d88be98548c6ed860970)) + + +### Bug Fixes + +* fix jobname checking logic ([1358f5f](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/1358f5fd3070cf1c4f0b08e65b9cd805dd8a1e90)) +* jobname checking ([f7a67d4](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/f7a67d4d6dfee279fa1ff088e1d1f9241dfdcbe0)) +* remove superfluous attribute ([7d283f8](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/7d283f8551f0d2c80dadf87717d804422b3b5c09)) + + +### Performance Improvements + +* improve plugin lookup ([514514f](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/514514f22fcd3387915a65969ffd1d6c14c56964)) + + +### Miscellaneous Chores + +* release 1.0 ([59415f4](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/59415f461616ab69668ea06c4a34932de70ea4bc)) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b39b46c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Snakemake Community + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..bef884c --- /dev/null +++ b/README.md @@ -0,0 +1,152 @@ +# Stable interfaces and functionality for Snakemake executor plugins + +This package provides a stable interface for interactions between Snakemake and its executor plugins (WIP). + +Plugins should implement the following skeleton to comply with this interface. +It is recommended to use Snakemake's poetry plugin to set up this skeleton (and automated testing) within a python package, see https://github.com/snakemake/poetry-snakemake-plugin. + +```python +from dataclasses import dataclass, field +from typing import List, Generator, Optional +from snakemake_interface_executor_plugins.executors.base import SubmittedJobInfo +from snakemake_interface_executor_plugins.executors.remote import RemoteExecutor +from snakemake_interface_executor_plugins.settings import ( + ExecutorSettingsBase, CommonSettings +) +from snakemake_interface_executor_plugins.workflow import WorkflowExecutorInterface +from snakemake_interface_executor_plugins.logging import LoggerExecutorInterface +from snakemake_interface_executor_plugins.jobs import ( + JobExecutorInterface, +) + +# Optional: +# Define additional settings for your executor. +# They will occur in the Snakemake CLI as --- +# Omit this class if you don't need any. +# Make sure that all defined fields are Optional and specify a default value +# of None or anything else that makes sense in your case. +@dataclass +class ExecutorSettings(ExecutorSettingsBase): + myparam: Optional[int] = field( + default=None, + metadata={ + "help": "Some help text", + # Optionally request that setting is also available for specification + # via an environment variable. The variable will be named automatically as + # SNAKEMAKE__, all upper case. + # This mechanism should only be used for passwords and usernames. + # For other items, we rather recommend to let people use a profile + # for setting defaults + # (https://snakemake.readthedocs.io/en/stable/executing/cli.html#profiles). + "env_var": False, + # Optionally specify a function that parses the value given by the user. + # This is useful to create complex types from the user input. + "parse_func": ..., + # If a parse_func is specified, you also have to specify an unparse_func + # that converts the parsed value back to a string. + "unparse_func": ..., + # Optionally specify that setting is required when the executor is in use. + "required": True, + # Optionally specify multiple args with "nargs": True + }, + ) + + +# Required: +# Specify common settings shared by various executors. +common_settings = CommonSettings( + # define whether your executor plugin executes locally + # or remotely. In virtually all cases, it will be remote execution + # (cluster, cloud, etc.). Only Snakemake's standard execution + # plugins (snakemake-executor-plugin-dryrun, snakemake-executor-plugin-local) + # are expected to specify False here. + non_local_exec=True, + # Whether the executor implies to not have a shared file system + implies_no_shared_fs=True, + # whether to deploy workflow sources to default storage provider before execution + job_deploy_sources=True, + # whether arguments for setting the storage provider shall be passed to jobs + pass_default_storage_provider_args=True, + # whether arguments for setting default resources shall be passed to jobs + pass_default_resources_args=True, + # whether environment variables shall be passed to jobs (if False, use + # self.envvars() to obtain a dict of environment variables and their values + # and pass them e.g. as secrets to the execution backend) + pass_envvar_declarations_to_cmd=True, + # whether the default storage provider shall be deployed before the job is run on + # the remote node. Usually set to True if the executor does not assume a shared fs + auto_deploy_default_storage_provider=True, + # specify initial amount of seconds to sleep before checking for job status + init_seconds_before_status_checks=0, +) + + +# Required: +# Implementation of your executor +class Executor(RemoteExecutor): + def __post_init__(self): + # access workflow + self.workflow + # access executor specific settings + self.workflow.executor_settings + + # IMPORTANT: in your plugin, only access methods and properties of + # Snakemake objects (like Workflow, Persistence, etc.) that are + # defined in the interfaces found in the + # snakemake-interface-executor-plugins and the + # snakemake-interface-common package. + # Other parts of those objects are NOT guaranteed to remain + # stable across new releases. + + # To ensure that the used interfaces are not changing, you should + # depend on these packages as >=a.b.c, Generator[SubmittedJobInfo, None, None]: + # Check the status of active jobs. + + # You have to iterate over the given list active_jobs. + # If you provided it above, each will have its external_jobid set according + # to the information you provided at submission time. + # For jobs that have finished successfully, you have to call + # self.report_job_success(active_job). + # For jobs that have errored, you have to call + # self.report_job_error(active_job). + # This will also take care of providing a proper error message. + # Usually there is no need to perform additional logging here. + # Jobs that are still running have to be yielded. + # + # For queries to the remote middleware, please use + # self.status_rate_limiter like this: + # + # async with self.status_rate_limiter: + # # query remote middleware here + # + # To modify the time until the next call of this method, + # you can set self.next_sleep_seconds here. + ... + + def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]): + # Cancel all active jobs. + # This method is called when Snakemake is interrupted. + ... +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e590771 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +[tool.poetry] +authors = ["Johannes Köster "] +description = "This package provides a stable interface for interactions between Snakemake and its executor plugins." +license = "MIT" +name = "snakemake-interface-executor-plugins" +packages = [{include = "snakemake_interface_executor_plugins"}] +readme = "README.md" +version = "9.3.0" + +[tool.poetry.dependencies] +argparse-dataclass = "^2.0.0" +python = "^3.11" +throttler = "^1.2.2" +snakemake-interface-common = "^1.17.4" + +[tool.poetry.dev-dependencies] +black = "^24.0.0" +coverage = {extras = ["toml"], version = "^6.3.1"} +flake8 = "^4.0.1" +flake8-bugbear = "^22.1.11" +pytest = "^7.0" +snakemake = {git="https://github.com/snakemake/snakemake.git"} +snakemake-executor-plugin-cluster-generic = {git = "https://github.com/snakemake/snakemake-executor-plugin-cluster-generic.git"} + +[tool.coverage.run] +omit = [".*", "*/site-packages/*"] + +[tool.coverage.report] +fail_under = 63 + +[build-system] +build-backend = "poetry.core.masonry.api" +requires = ["poetry-core"] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..994716a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,7 @@ +[flake8] +# Recommend matching the black line length (default 88), +# rather than using the flake8 default of 79: +max-line-length = 106 +extend-ignore = + # See https://github.com/PyCQA/pycodestyle/issues/373 + E203, \ No newline at end of file diff --git a/snakemake_interface_software_deployment_plugins/__init__.py b/snakemake_interface_software_deployment_plugins/__init__.py new file mode 100644 index 0000000..4c7ff7e --- /dev/null +++ b/snakemake_interface_software_deployment_plugins/__init__.py @@ -0,0 +1,118 @@ +__author__ = "Johannes Köster" +__copyright__ = "Copyright 2024, Johannes Köster" +__email__ = "johannes.koester@uni-due.de" +__license__ = "MIT" + +from abc import ABC, abstractmethod +import hashlib +from pathlib import Path +from typing import List, Optional, Self + +from snakemake_interface_software_deployment_plugins.settings import SoftwareDeploymentProviderSettingsBase + + +class EnvSpecBase(ABC): + pass + + +class EnvBase(ABC): + def __init__(self, provider: "SoftwareDeploymentProviderBase", spec: EnvSpecBase, parent_env: Optional[Self]): + self.provider = provider + self.spec = spec + self.parent_env = parent_env + self._managed_hash_store = None + self._managed_deployment_hash_store = None + self.__post_init__() + + def __post_init__(self): # noqa B027 + """Do stuff after object initialization.""" + pass + + @abstractmethod + def decorate_shellcmd(self, cmd: str) -> str: + """Decorate given shell command such that it runs within the environment.""" + ... + + @abstractmethod + def deploy(self): + """Deploy the environment to self.provider.deployment_path. + + When issuing shell commands, the environment should use + self.managed_decorate_shellcmd in order to ensure that it runs within eventual + parent environments (e.g. a container or an env module). + """ + ... + + @abstractmethod + def archive(self): + """Archive the environment to self.provider.archive_path. + + When issuing shell commands, the environment should use + self.managed_decorate_shellcmd in order to ensure that it runs within eventual + parent environments (e.g. a container or an env module). + """ + ... + + @abstractmethod + def hash(self, hash_object) -> None: + """Update given hash such that it changes whenever the environment + could potentially contain a different set of software (in terms of versions or + packages). + """ + ... + + @abstractmethod + def deployment_hash(self, hash_object): + """Update given hash such that it changes whenever the environment + needs to be redeployed, e.g. because its content has changed or the + deployment location has changed. The latter is only relevant if the + deployment is senstivive to the path (e.g. in case of conda, which patches + the RPATH in binaries). + """ + ... + + def managed_decorate_shellcmd(self, cmd: str) -> str: + cmd = self.decorate_shellcmd(cmd) + if self.parent_env is not None: + cmd = self.parent_env.managed_decorate_shellcmd(cmd) + return cmd + + def managed_hash(self) -> str: + return self._managed_generic_hash("hash") + + def managed_deployment_hash(self) -> str: + return self._managed_generic_hash("deployment_hash") + + def _managed_generic_hash(self, kind: str) -> str: + store = getattr(self, f"_managed_{kind}_store") + if store is None: + hash_object = hashlib.md5() + if self.parent_env is not None: + hash_object.update(getattr(self.parent_env, kind)().encode()) + getattr(self, kind)(hash_object) + store = hash_object.hexdigest() + return store + + +class SoftwareDeploymentProviderBase(ABC): + def __init__( + self, + name: str, + prefix: Path, + settings: Optional[SoftwareDeploymentProviderSettingsBase] = None, + ): + self.settings = settings + self.deployment_path = prefix / name + self.archive_path = prefix / f"{name}-archive" + self.__post_init__() + + def __post_init__(self): # noqa B027 + pass + + @classmethod + @abstractmethod + def get_env_cls(cls) -> EnvBase: + ... + + def env(self, spec: EnvSpecBase) -> EnvBase: + return self.get_env_cls()(self, spec) \ No newline at end of file diff --git a/snakemake_interface_software_deployment_plugins/_common.py b/snakemake_interface_software_deployment_plugins/_common.py new file mode 100644 index 0000000..3553b2f --- /dev/null +++ b/snakemake_interface_software_deployment_plugins/_common.py @@ -0,0 +1,8 @@ +__author__ = "Johannes Köster" +__copyright__ = "Copyright 2024, Johannes Köster" +__email__ = "johannes.koester@uni-due.de" +__license__ = "MIT" + + +software_deployment_plugin_prefix = "snakemake-software-deployment-plugin-" +software_deployment_plugin_module_prefix = software_deployment_plugin_prefix.replace("-", "_") diff --git a/snakemake_interface_software_deployment_plugins/registry/__init__.py b/snakemake_interface_software_deployment_plugins/registry/__init__.py new file mode 100644 index 0000000..00c2c08 --- /dev/null +++ b/snakemake_interface_software_deployment_plugins/registry/__init__.py @@ -0,0 +1,56 @@ +__author__ = "Johannes Köster" +__copyright__ = "Copyright 2024, Johannes Köster" +__email__ = "johannes.koester@uni-due.de" +__license__ = "MIT" + +import types +from typing import Mapping +from snakemake_interface_software_deployment_plugins.settings import ( + CommonSettings, + SoftwareDeploymentProviderSettingsBase, +) + +from snakemake_interface_common.plugin_registry.attribute_types import ( + AttributeKind, + AttributeMode, + AttributeType, +) +from snakemake_interface_software_deployment_plugins.registry.plugin import Plugin +from snakemake_interface_common.plugin_registry import PluginRegistryBase +from snakemake_interface_software_deployment_plugins import SoftwareDeploymentProviderBase, _common as common + + +class SoftwareDeploymentPluginRegistry(PluginRegistryBase): + """This class is a singleton that holds all registered executor plugins.""" + + @property + def module_prefix(self) -> str: + return common.software_deployment_plugin_module_prefix + + def load_plugin(self, name: str, module: types.ModuleType) -> Plugin: + """Load a plugin by name.""" + return Plugin( + _name=name, + _software_deployment_provider=module.SoftwareDeploymentProvider, + common_settings=module.common_settings, + _software_deployment_settings_cls=getattr(module, "SoftwareDeploymentProviderSettings", None), + ) + + def expected_attributes(self) -> Mapping[str, AttributeType]: + return { + "common_settings": AttributeType( + cls=CommonSettings, + mode=AttributeMode.REQUIRED, + kind=AttributeKind.OBJECT, + ), + "SoftwareDeploymentProviderSettings": AttributeType( + cls=SoftwareDeploymentProviderSettingsBase, + mode=AttributeMode.OPTIONAL, + kind=AttributeKind.CLASS, + ), + "SoftwareDeploymentProvider": AttributeType( + cls=SoftwareDeploymentProviderBase, + mode=AttributeMode.REQUIRED, + kind=AttributeKind.CLASS, + ), + } diff --git a/snakemake_interface_software_deployment_plugins/registry/plugin.py b/snakemake_interface_software_deployment_plugins/registry/plugin.py new file mode 100644 index 0000000..0a72fef --- /dev/null +++ b/snakemake_interface_software_deployment_plugins/registry/plugin.py @@ -0,0 +1,42 @@ +__author__ = "Johannes Köster" +__copyright__ = "Copyright 2024, Johannes Köster" +__email__ = "johannes.koester@uni-due.de" +__license__ = "MIT" + +from dataclasses import dataclass +from typing import Optional, Type +from snakemake_interface_software_deployment_plugins.settings import ( + CommonSettings, + SoftwareDeploymentSettingsBase, +) +import snakemake_interface_software_deployment_plugins._common as common + +from snakemake_interface_common.plugin_registry.plugin import PluginBase + + +@dataclass +class Plugin(PluginBase): + _software_deployment_provider: object + common_settings: CommonSettings + _software_deployment_settings_cls: Optional[Type[SoftwareDeploymentSettingsBase]] + _name: str + + @property + def software_deployment_provider(self): + return self._software_deployment_provider(name=self._only_name) + + @property + def name(self): + return self._name + + @property + def cli_prefix(self): + return "sdm-" + self._only_name + + @property + def _only_name(self): + return self.name.replace(common.software_deployment_plugin_module_prefix, "") + + @property + def settings_cls(self): + return self._software_deployment_settings_cls diff --git a/snakemake_interface_software_deployment_plugins/settings.py b/snakemake_interface_software_deployment_plugins/settings.py new file mode 100644 index 0000000..676d168 --- /dev/null +++ b/snakemake_interface_software_deployment_plugins/settings.py @@ -0,0 +1,36 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List, Sequence, Set + +from snakemake_interface_common.settings import SettingsEnumBase, TSettingsEnumBase + + +import snakemake_interface_common.plugin_registry.plugin + + +@dataclass +class CommonSettings: + """Common Snakemake settings shared between executors that can be specified + by executor plugins. + + The plugin has to specify an instance of this class as the value of the + common_settings attribute. + + Attributes + ---------- + + """ + pass + + +@dataclass +class SoftwareDeploymentProviderSettingsBase( + snakemake_interface_common.plugin_registry.plugin.SettingsBase +): + """Base class for software deployment settings. + + Software deployment plugins can define a subclass of this class, + named 'SoftwareDeploymentProviderSettings'. + """ + + pass diff --git a/tests/test_py37.py b/tests/test_py37.py new file mode 100644 index 0000000..3de586a --- /dev/null +++ b/tests/test_py37.py @@ -0,0 +1,8 @@ +import sys + + +def test_imports_from_py37(): + assert sys.version_info >= (3, 7) and sys.version_info < (3, 8) + from snakemake_interface_software_deployment_plugins import ( # noqa: F401 + settings, + ) diff --git a/tests/tests.py b/tests/tests.py new file mode 100644 index 0000000..af2f841 --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,56 @@ +from typing import List +from snakemake_interface_software_deployment_plugins.registry import SoftwareDeploymentPluginRegistry +from snakemake_interface_common.plugin_registry.tests import TestRegistryBase +from snakemake_interface_common.plugin_registry.plugin import PluginBase, SettingsBase +from snakemake_interface_common.plugin_registry import PluginRegistryBase +from snakemake_interface_software_deployment_plugins.utils import format_cli_arg + + +class TestRegistry(TestRegistryBase): + __test__ = True + + def get_registry(self) -> PluginRegistryBase: + # ensure that the singleton is reset + SoftwareDeploymentPluginRegistry._instance = None + return SoftwareDeploymentPluginRegistry() + + def get_test_plugin_name(self) -> str: + return "cluster-generic" + + def validate_plugin(self, plugin: PluginBase): + assert plugin._executor_settings_cls is not None + assert plugin.common_settings.non_local_exec is True + assert plugin.executor is not None + + def validate_settings(self, settings: SettingsBase, plugin: PluginBase): + assert isinstance(settings, plugin._executor_settings_cls) + + def get_example_args(self) -> List[str]: + return ["--cluster-generic-submit-cmd", "qsub"] + + +def test_format_cli_arg_single_quote(): + fmt = format_cli_arg("--default-resources", {"slurm_extra": "'--gres=gpu:1'"}) + assert fmt == "--default-resources \"slurm_extra='--gres=gpu:1'\"" + + +def test_format_cli_arg_double_quote(): + fmt = format_cli_arg("--default-resources", {"slurm_extra": '"--gres=gpu:1"'}) + assert fmt == "--default-resources 'slurm_extra=\"--gres=gpu:1\"'" + + +def test_format_cli_arg_int(): + fmt = format_cli_arg("--default-resources", {"mem_mb": 200}) + assert fmt == "--default-resources 'mem_mb=200'" + + +def test_format_cli_arg_expr(): + fmt = format_cli_arg( + "--default-resources", {"mem_mb": "min(2 * input.size_mb, 2000)"} + ) + assert fmt == "--default-resources 'mem_mb=min(2 * input.size_mb, 2000)'" + + +def test_format_cli_arg_list(): + fmt = format_cli_arg("--config", ["foo={'bar': 1}"]) + assert fmt == "--config \"foo={'bar': 1}\""