Skip to content

Commit e3f6cf7

Browse files
mr-cmr-c
committed
build binary wheels
But don't publish them yet
1 parent 8dee8e9 commit e3f6cf7

9 files changed

+265
-14
lines changed

‎.circleci/config.yml

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
version: 2.1
2+
3+
parameters:
4+
REF:
5+
type: string
6+
default: ''
7+
description: Optional tag to build
8+
9+
jobs:
10+
arm-wheels:
11+
parameters:
12+
build:
13+
type: string
14+
image:
15+
type: string
16+
17+
machine:
18+
image: ubuntu-2204:current
19+
resource_class: arm.medium # two vCPUs
20+
21+
environment:
22+
CIBW_ARCHS: "aarch64"
23+
CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>"
24+
CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>"
25+
CIBW_BUILD: "<< parameters.build >>"
26+
27+
steps:
28+
- checkout
29+
- when:
30+
condition: << pipeline.parameters.REF >>
31+
steps:
32+
- run:
33+
name: Checkout branch/tag << pipeline.parameters.REF >>
34+
command: |
35+
echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists"
36+
git checkout << pipeline.parameters.REF >> || true
37+
git pull origin << pipeline.parameters.REF >> || true
38+
- run:
39+
name: install cibuildwheel and other build reqs
40+
command: |
41+
python3 -m pip install --upgrade pip setuptools setuptools_scm[toml]
42+
python3 -m pip install -rcibw-requirements.txt
43+
44+
- run:
45+
name: pip freeze
46+
command: |
47+
python3 -m pip freeze
48+
49+
- run:
50+
name: list wheels
51+
command: |
52+
python3 -m cibuildwheel . --print-build-identifiers
53+
54+
- run:
55+
name: cibuildwheel
56+
command: |
57+
python3 -m cibuildwheel .
58+
59+
- store_test_results:
60+
path: test-results/
61+
62+
- store_artifacts:
63+
path: wheelhouse/
64+
65+
# - when:
66+
# condition:
67+
# or:
68+
# - matches:
69+
# pattern: ".+"
70+
# value: "<< pipeline.git.tag >>"
71+
# - << pipeline.parameters.REF >>
72+
# steps:
73+
# - run:
74+
# environment:
75+
# TWINE_NONINTERACTIVE: "1"
76+
# command: |
77+
# python3 -m pip install twine
78+
# python3 -m twine upload --verbose --skip-existing wheelhouse/*
79+
80+
workflows:
81+
wheels: # This is the name of the workflow, feel free to change it to better match your workflow.
82+
# Inside the workflow, you define the jobs you want to run.
83+
jobs:
84+
- arm-wheels:
85+
name: arm-wheels-manylinux_2_28
86+
filters:
87+
tags:
88+
only: /.*/
89+
build: "*manylinux*"
90+
image: quay.io/pypa/manylinux_2_28_aarch64
91+
- arm-wheels:
92+
name: arm-wheels-musllinux_1_1
93+
filters:
94+
tags:
95+
only: /.*/
96+
build: "*musllinux*"
97+
image: quay.io/pypa/musllinux_1_1_aarch64
98+
- arm-wheels:
99+
name: arm-wheels-musllinux_1_2
100+
filters:
101+
tags:
102+
only: /.*/
103+
build: "*musllinux*"
104+
image: quay.io/pypa/musllinux_1_2_aarch64

‎.github/workflows/ci-tests.yml

+8-8
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,11 @@ jobs:
4444
with:
4545
fetch-depth: 0
4646

47-
- name: Set up Singularity
47+
- name: Set up Singularity and environment-modules
4848
if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
4949
run: |
5050
wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
51-
sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
51+
sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules
5252
5353
- name: Give the test runner user a name to make provenance happy.
5454
if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
@@ -132,10 +132,10 @@ jobs:
132132
with:
133133
fetch-depth: 0
134134

135-
- name: Set up Singularity
135+
- name: Set up Singularity and environment-modules
136136
run: |
137137
wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
138-
sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
138+
sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules
139139
140140
- name: Give the test runner user a name to make provenance happy.
141141
run: sudo usermod -c 'CI Runner' "$(whoami)"
@@ -180,11 +180,11 @@ jobs:
180180
steps:
181181
- uses: actions/checkout@v4
182182

183-
- name: Set up Singularity
183+
- name: Set up Singularity and environment-modules
184184
if: ${{ matrix.container == 'singularity' }}
185185
run: |
186186
wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
187-
sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
187+
sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules
188188
189189
- name: Singularity cache
190190
if: ${{ matrix.container == 'singularity' }}
@@ -229,10 +229,10 @@ jobs:
229229
steps:
230230
- uses: actions/checkout@v4
231231

232-
- name: Set up Singularity
232+
- name: Set up Singularity and environment-modules
233233
run: |
234234
wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
235-
sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
235+
sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules
236236
237237
- name: Set up Python
238238
uses: actions/setup-python@v5

‎.github/workflows/wheels.yml

+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: Python package build and publish
2+
3+
on:
4+
release:
5+
types: [published]
6+
workflow_dispatch: {}
7+
repository_dispatch: {}
8+
pull_request:
9+
push:
10+
branches:
11+
- main
12+
13+
concurrency:
14+
group: wheels-${{ github.event.pull_request.number || github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
build_wheels:
19+
name: ${{ matrix.image }} wheels
20+
runs-on: ubuntu-24.04
21+
strategy:
22+
matrix:
23+
include:
24+
- image: manylinux_2_28_x86_64
25+
build: "*manylinux*"
26+
- image: musllinux_1_1_x86_64
27+
build: "*musllinux*"
28+
- image: musllinux_1_2_x86_64
29+
build: "*musllinux*"
30+
31+
steps:
32+
- uses: actions/checkout@v4
33+
if: ${{ github.event_name != 'repository_dispatch' }}
34+
with:
35+
fetch-depth: 0 # slow, but gets all the tags
36+
- uses: actions/checkout@v4
37+
if: ${{ github.event_name == 'repository_dispatch' }}
38+
with:
39+
fetch-depth: 0 # slow, but gets all the tags
40+
ref: ${{ github.event.client_payload.ref }}
41+
42+
# - name: Set up QEMU
43+
# if: runner.os == 'Linux'
44+
# uses: docker/setup-qemu-action@v2
45+
# with:
46+
# platforms: all
47+
48+
- name: Build wheels
49+
uses: pypa/[email protected]
50+
env:
51+
CIBW_BUILD: ${{ matrix.build }}
52+
CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }}
53+
CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }}
54+
# configure cibuildwheel to build native 64-bit archs ('auto64'), and some
55+
# emulated ones
56+
# Linux arm64 wheels are built on circleci
57+
CIBW_ARCHS_LINUX: auto64 # ppc64le s390x
58+
59+
- uses: actions/upload-artifact@v4
60+
with:
61+
name: artifact-${{ matrix.image }}
62+
path: ./wheelhouse/*.whl
63+
64+
build_sdist:
65+
name: Build source distribution
66+
runs-on: ubuntu-24.04
67+
steps:
68+
- uses: actions/checkout@v4
69+
if: ${{ github.event_name != 'repository_dispatch' }}
70+
with:
71+
fetch-depth: 0 # slow, but gets all the tags
72+
- uses: actions/checkout@v4
73+
if: ${{ github.event_name == 'repository_dispatch' }}
74+
with:
75+
fetch-depth: 0 # slow, but gets all the tags
76+
ref: ${{ github.event.client_payload.ref }}
77+
78+
- name: Build sdist
79+
run: pipx run build --sdist
80+
81+
- uses: actions/upload-artifact@v4
82+
with:
83+
name: artifact-source
84+
path: dist/*.tar.gz
85+
86+
build_wheels_macos:
87+
name: Build wheels on ${{ matrix.os }}
88+
runs-on: ${{ matrix.os }}
89+
strategy:
90+
matrix:
91+
# macos-13 is an intel runner, macos-14 is apple silicon
92+
os: [macos-13, macos-14]
93+
steps:
94+
- uses: actions/checkout@v4
95+
if: ${{ github.event_name != 'repository_dispatch' }}
96+
with:
97+
fetch-depth: 0 # slow, but gets all the tags
98+
- uses: actions/checkout@v4
99+
if: ${{ github.event_name == 'repository_dispatch' }}
100+
with:
101+
fetch-depth: 0 # slow, but gets all the tags
102+
ref: ${{ github.event.client_payload.ref }}
103+
104+
- name: Build wheels
105+
uses: pypa/[email protected]
106+
107+
- uses: actions/upload-artifact@v4
108+
with:
109+
name: artifact-${{ matrix.os }}-${{ strategy.job-index }}
110+
path: ./wheelhouse/*.whl
111+
112+
# upload_pypi:
113+
# needs: [build_wheels, build_sdist]
114+
# runs-on: ubuntu-24.04
115+
# environment: deploy
116+
# permissions:
117+
# id-token: write
118+
# if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true)
119+
# steps:
120+
# - uses: actions/download-artifact@v4
121+
# with:
122+
# # unpacks default artifact into dist/
123+
# pattern: artifact-*
124+
# merge-multiple: true
125+
# path: dist
126+
127+
# - name: Publish package distributions to PyPI
128+
# uses: pypa/gh-action-pypi-publish@release/v1
129+
# with:
130+
# skip-existing: true

‎MANIFEST.in

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ include tests/reloc/dir2/*
1919
include tests/checker_wf/*
2020
include tests/subgraph/*
2121
include tests/input_deps/*
22+
recursive-include tests/test_deps_env
2223
include tests/trs/*
2324
include tests/wf/generator/*
2425
include cwltool/py.typed

‎cibw-requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
cibuildwheel==2.21.3

‎cwltool/software_requirements.py

+2
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ class DependenciesConfiguration:
5050

5151
def __init__(self, args: argparse.Namespace) -> None:
5252
"""Initialize."""
53+
self.tool_dependency_dir: Optional[str] = None
54+
self.dependency_resolvers_config_file: Optional[str] = None
5355
conf_file = getattr(args, "beta_dependency_resolvers_configuration", None)
5456
tool_dependency_dir = getattr(args, "beta_dependencies_directory", None)
5557
conda_dependencies = getattr(args, "beta_conda_dependencies", None)

‎pyproject.toml

+13
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,19 @@ build-backend = "setuptools.build_meta"
1717
[tool.setuptools_scm]
1818
write_to = "cwltool/_version.py"
1919

20+
[tool.cibuildwheel]
21+
test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool"
22+
test-requires = "-r test-requirements.txt"
23+
test-extras = "deps"
24+
skip = "pp*"
25+
# ^ skip building wheels on PyPy (any version)
26+
build-verbosity = 1
27+
environment = { CWLTOOL_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" }
28+
29+
# Install system library
30+
[tool.cibuildwheel.linux]
31+
before-all = "apk add libxml2-dev libxslt-dev nodejs || yum install -y libxml2-devel libxslt-devel nodejs environment-modules || apt-get install -y --no-install-recommends libxml2-dev libxslt-dev nodejs environment-modules"
32+
2033
[tool.black]
2134
line-length = 100
2235
target-version = [ "py39" ]

‎test-requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ pytest>= 6.2,< 8.4
33
pytest-xdist>=3.2.0 # for the worksteal scheduler
44
psutil # enhances pytest-xdist to allow "-n logical"
55
pytest-httpserver
6-
pytest-retry;python_version>'3.9'
6+
pytest-retry;python_version>='3.9'
77
mock>=2.0.0
88
pytest-mock>=1.10.0
99
pytest-cov

‎tests/test_dependencies.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -119,15 +119,15 @@ def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
119119
"""Do a basic smoke test using environment modules to satisfy a SoftwareRequirement."""
120120
wflow = get_data("tests/random_lines.cwl")
121121
job = get_data("tests/random_lines_job.json")
122-
monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles"))
122+
monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles"))
123123
error_code, _, stderr = get_main_output(
124124
[
125125
"--outdir",
126126
str(tmp_path / "out"),
127-
"--beta-dependency-resolvers-configuration",
128127
"--beta-dependencies-directory",
129128
str(tmp_path / "deps"),
130-
"tests/test_deps_env_modules_resolvers_conf.yml",
129+
"--beta-dependency-resolvers-configuration",
130+
get_data("tests/test_deps_env_modules_resolvers_conf.yml"),
131131
"--debug",
132132
wflow,
133133
job,
@@ -145,7 +145,7 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
145145
146146
Do so by by running `env` as the tool and parsing its output.
147147
"""
148-
monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles"))
148+
monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles"))
149149
tool_env = get_tool_env(
150150
tmp_path,
151151
[
@@ -155,6 +155,6 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
155155
get_data("tests/env_with_software_req.yml"),
156156
)
157157

158-
assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space "
158+
assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space ", tool_env
159159
tool_path = tool_env["PATH"].split(":")
160160
assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path

0 commit comments

Comments
 (0)