Skip to content

Commit 8000775

Browse files
oscarbenjaminoscarbenjamin
committed
maint: build+test wheels for Windows in CI
1 parent edf6be7 commit 8000775

9 files changed

+305
-12
lines changed

‎.github/workflows/buildwheel.yml

+24-11
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,7 @@ jobs:
1010
strategy:
1111
fail-fast: false
1212
matrix:
13-
# os: [ubuntu-20.04, windows-2019, macos-12]
14-
os: [ubuntu-20.04, macos-12]
13+
os: [ubuntu-20.04, windows-2019, macos-12]
1514

1615
steps:
1716
- uses: actions/checkout@v3
@@ -20,22 +19,38 @@ jobs:
2019
with:
2120
python-version: '3.10'
2221

22+
- uses: msys2/setup-msys2@v2
23+
with:
24+
msystem: mingw64
25+
# path-type inherit is used so that when cibuildwheel calls msys2 to
26+
# run bin/cibw_prepare_python_windows.sh the virtual environment
27+
# created by cibuildwheel will be available within msys2. The
28+
# msys2/setup-msys2 README warns that using inherit here can be
29+
# problematic in some situations. Maybe there is a better way to do
30+
# this.
31+
path-type: inherit
32+
if: ${{ matrix.os == 'windows-2019' }}
33+
2334
- name: Build wheels
2435
uses: pypa/[email protected]
2536
env:
2637
CIBW_BUILD: cp39-* cp310-* cp311-*
27-
#CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_*"
28-
CIBW_SKIP: "*-win32 *-musllinux_*"
38+
CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_*"
39+
#CIBW_SKIP: "*-win32 *-musllinux_*"
2940
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
3041
CIBW_MANYLINUX_I686_IMAGE: manylinux2014
3142
CIBW_BEFORE_ALL_LINUX: bin/cibw_before_build_linux.sh
3243
CIBW_BEFORE_ALL_MACOS: bin/cibw_before_build_macosx.sh
33-
CIBW_BEFORE_BUILD: pip install numpy cython
44+
CIBW_BEFORE_ALL_WINDOWS: msys2 -c bin/cibw_before_build_windows.sh
45+
CIBW_BEFORE_BUILD_WINDOWS: msys2 -c bin/cibw_prepare_python_windows.sh
46+
CIBW_BEFORE_BUILD: pip install numpy cython delvewheel
3447
CIBW_ENVIRONMENT: >
3548
C_INCLUDE_PATH=$(pwd)/.local/include/
3649
LIBRARY_PATH=$(pwd)/.local/lib/
3750
LD_LIBRARY_PATH=$(pwd)/.local/lib:$LD_LIBRARY_PATH
38-
CIBW_TEST_COMMAND: python -c 'import flint; print(str(flint.fmpz(2)))'
51+
PYTHON_FLINT_MINGW64=true
52+
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bin\repair_mingw_wheel.bat {dest_dir} {wheel}
53+
CIBW_TEST_COMMAND: python -c "import flint; print(str(flint.fmpz(2)))"
3954

4055
- uses: actions/upload-artifact@v3
4156
with:
@@ -48,7 +63,7 @@ jobs:
4863
strategy:
4964
fail-fast: false
5065
matrix:
51-
os: [ubuntu-20.04, macos-12]
66+
os: [ubuntu-20.04, windows-2019, macos-12]
5267
python-version: ['3.9', '3.10', '3.11']
5368

5469
steps:
@@ -60,7 +75,5 @@ jobs:
6075
with:
6176
name: artifact
6277
path: wheelhouse
63-
- run: python -m venv venv
64-
- run: venv/bin/pip install -U pip
65-
- run: venv/bin/pip install --find-links wheelhouse python_flint
66-
- run: venv/bin/python test/test.py
78+
- run: pip install --find-links wheelhouse python_flint
79+
- run: python test/test.py

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ __pycache__
1111
MANIFEST
1212
.eggs
1313
.local
14+
*.egg-info

‎bin/build_mingw64_wheel.sh

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#!/bin/bash
2+
#
3+
# make_wheels.sh
4+
#
5+
# Build relocatable Windows wheel for python_flint using the mingw-w64
6+
# toolchain in the MSYS2 enironment.
7+
#
8+
# - First install Python
9+
#
10+
# https://www.python.org/ftp/python/3.10.8/python-3.10.8-amd64.exe
11+
#
12+
# - Then checkout the code:
13+
#
14+
# $ git clone https://github.com/fredrik-johansson/python-flint/issues/1
15+
#
16+
# - Then install msys2
17+
#
18+
# https://repo.msys2.org/distrib/x86_64/msys2-x86_64-20221028.exe
19+
#
20+
# - Then open msys2, cd into the checked out repo. Make sure setup.py says
21+
#
22+
# libraries = ["arb", "flint", "mpfr", "gmp"]
23+
#
24+
# - Set the environment variable to the directory containing the installed
25+
# Python that we want to build a wheel for i.e. the one installed from
26+
# python.org. If python was on PATH then it would be
27+
#
28+
# PYTHONDIR=`dirname $(which python)`
29+
# PYTHONVER=3.10
30+
#
31+
# - Then run this script.
32+
33+
set -o errexit
34+
35+
#
36+
# In CI this environment variable needs to be set to the directory containing
37+
# the python.org installation of Python. If Python is installed in msys2 then
38+
# it is also necesary to set this environment variable so that it picks up the
39+
# right installation of Python i.e. the one that we want to build a wheel for.
40+
#
41+
if [[ -z "$PYTHONDIR" ]]; then
42+
PYTHONDIR=`dirname $(which python)`
43+
fi
44+
PYTHON=$PYTHONDIR/python
45+
VER="${PYTHONVER//./}"
46+
47+
WHEELNAME=python_flint-0.3.0-cp$VER-cp$VER-win_amd64.whl
48+
49+
$PYTHON -c 'print("hello world")'
50+
51+
echo $PYTHONDIR
52+
ls $PYTHONDIR
53+
ls $PYTHONDIR/libs
54+
55+
# Install the mingw-w64 toolchain
56+
pacman -S --noconfirm mingw-w64-x86_64-gcc m4 make mingw-w64-x86_64-tools-git
57+
58+
# This takes ~30mins
59+
#bin/build_dependencies_unix.sh
60+
61+
# Add the libpython$VER.a file to Python installation
62+
cd $PYTHONDIR
63+
gendef python$VER.dll
64+
dlltool --dllname python$VER.dll --def python$VER.def --output-lib libpython$VER.a
65+
mv libpython$VER.a libs
66+
cd -
67+
68+
# Make a virtual environment to install the build dependencies
69+
$PYTHON -m venv .local/venv
70+
source .local/venv/Scripts/activate
71+
pip install numpy cython wheel delvewheel psutil
72+
73+
# Pass this flag to setup.py
74+
export PYTHON_FLINT_MINGW64_TMP=true
75+
76+
# Build the wheel
77+
C_INCLUDE_PATH=.local/include/ LIBRARY_PATH=.local/lib/ python setup.py build_ext -cmingw32 -f bdist_wheel
78+
79+
# delvewheel requires DLLs created by mingw64 to be stripped
80+
#
81+
# https://github.com/scipy/scipy/blob/main/tools/wheels/repair_windows.sh
82+
strip .local/bin/*.dll .local/lib/*.dll
83+
84+
# Unpack the wheel and strip the .pyd DLL inside
85+
cd dist
86+
wheel unpack $WHEELNAME
87+
rm $WHEELNAME
88+
strip python_flint-*/flint/*.pyd
89+
wheel pack python_flint-*
90+
cd ..
91+
92+
# Make the wheel relocatable
93+
delvewheel repair dist/python_flint-0.3.0-cp$VER-cp$VER-win_amd64.whl \
94+
--add-path .local/bin:.local/lib/
95+
96+
# Make a virtual enironment to test the wheel
97+
$PYTHON -m venv test_env
98+
source test_env/Scripts/activate
99+
pip install wheelhouse/$WHEELNAME
100+
python -c 'import flint; print(flint.fmpz(2) + 2)' # 2 + 2 = 4?

‎bin/cibw.bat

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
rem
2+
rem This bat file can be used to test cibuildwheel locally on Windows. The
3+
rem cibw_*_windows.sh files have lines to set the PATH for working locally but
4+
rem those are commented out because they are not needed in CI. To use this
5+
rem script
6+
rem
7+
rem 1. Uncomment those lines
8+
rem 2. > pip install cibuildwheel
9+
rem 3. > bin\cibw.bat
10+
rem
11+
rem The variables defined below should match those that are set in CI except
12+
rem that C:\msys64\usr\bin\bash should just be msys2 -c in CI.
13+
rem
14+
rem It is also worth commenting out the line to build GMP etc after you have
15+
rem built those once because that is by far the slowest step.
16+
rem
17+
18+
rem
19+
rem If this script is run repeatedly then it would fail because of any leftover
20+
rem wheels from a previous run so we delete them here.
21+
rem
22+
del /q wheelhouse\*
23+
24+
set CIBW_BUILD=cp39-* cp310-* cp311-*
25+
set CIBW_SKIP=*-win32 *-manylinux_i686 *-musllinux_*
26+
set CIBW_BEFORE_ALL_WINDOWS=C:\msys64\usr\bin\bash bin/cibw_before_build_windows.sh
27+
set CIBW_BEFORE_BUILD_WINDOWS=C:\msys64\usr\bin\bash bin/cibw_prepare_python_windows.sh
28+
set CIBW_ENVIRONMENT=PYTHON_FLINT_MINGW64=true
29+
set CIBW_REPAIR_WHEEL_COMMAND_WINDOWS=bin\repair_mingw_wheel.bat {dest_dir} {wheel}
30+
set CIBW_TEST_COMMAND=python -c "import flint; print(str(flint.fmpz(2)))"
31+
32+
cibuildwheel --platform windows

‎bin/cibw_before_build_windows.sh

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
5+
# Uncomment this to run cibuildwheel locally on Windows:
6+
# export PATH=$PATH:/c/msys64/usr/bin:/c/msys64/mingw64/bin
7+
8+
#
9+
# Make a setup.cfg to specify compiling with mingw64 (even though it says
10+
# mingw32...)
11+
#
12+
echo '[build]' > setup.cfg
13+
echo 'compiler = mingw32' >> setup.cfg
14+
cat setup.cfg
15+
16+
# Install the mingw-w64 toolchain
17+
pacman -S --noconfirm mingw-w64-x86_64-gcc m4 make mingw-w64-x86_64-tools-git
18+
19+
# This takes ~30mins
20+
bin/build_dependencies_unix.sh

‎bin/cibw_prepare_python_windows.sh

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
5+
# Uncomment this to run cibuildwheel locally on Windows:
6+
# export PATH=$PATH:/c/msys64/usr/bin:/c/msys64/mingw64/bin
7+
8+
# VER should be set be e.g. 310 for Python 3.10
9+
VER=`python -c 'import sys; print("%s%s" % sys.version_info[:2])'`
10+
echo VER=${VER}
11+
12+
###################################################
13+
# Find parent Python installation from the venv #
14+
###################################################
15+
16+
which python
17+
PYTHONBIN=`dirname $(which python)`
18+
PYTHONDIR=`dirname $PYTHONBIN`
19+
cfgfile=$PYTHONDIR/pyvenv.cfg
20+
homeline=`grep home $cfgfile`
21+
homepath=${homeline#*=}
22+
23+
echo ---------------------------------------------------
24+
echo $homepath
25+
echo ---------------------------------------------------
26+
27+
###################################################
28+
# Find pythonXX.dll and make a .a library #
29+
###################################################
30+
31+
cd $homepath
32+
gendef python${VER}.dll
33+
dlltool --dllname python${VER}.dll \
34+
--def python${VER}.def \
35+
--output-lib libpython${VER}.a
36+
37+
mv libpython${VER}.a libs
38+
39+
###################################################
40+
# Install build dependencies #
41+
###################################################
42+
43+
pip install Cython numpy delvewheel

‎bin/repair_mingw_wheel.bat

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
rem
2+
rem This batch file serves the purpose of taking Windows style path arguments,
3+
rem converting them to environment variables and calling msys2. This is needed
4+
rem because otherwise in CI msys2 -c will mangle the paths turning e.g. C:\a\b
5+
rem into C:ab.
6+
rem
7+
8+
set tempfile=tmpfile.deleteme
9+
set WHEELHOUSE=%1
10+
set WHEELNAME=%2
11+
12+
msys2 -c bin/repair_mingw_wheel.sh
13+
rem C:\msys64\usr\bin\bash bin/repair_mingw_wheel.sh

‎bin/repair_mingw_wheel.sh

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/bash
2+
#
3+
# Repair Windows wheels. See e.g.:
4+
#
5+
# https://github.com/scipy/scipy/blob/main/tools/wheels/repair_windows.sh
6+
7+
set -o errexit
8+
9+
# Uncomment this to run cibuildwheel locally on Windows:
10+
# export PATH=$PATH:/c/msys64/usr/bin:/c/msys64/mingw64/bin
11+
12+
# We cannot use ordinary command line arguments in CI because msys2 -c mangles
13+
# them. Instead we have a batch file to receive the arguments and convert them
14+
# into environment variables before calling this script. When running locally
15+
# this script could be run directly giving the parameters as command line
16+
# arguments instead.
17+
18+
if [[ -z "${WHEELHOUSE}" ]]; then
19+
WHEELNAME=$1
20+
fi
21+
if [[ -z "${WHEELNAME}" ]]; then
22+
WHEELHOUSE=$2
23+
fi
24+
25+
echo WHEELHOUSE=$WHEELHOUSE
26+
echo WHEELNAME=$WHEELNAME
27+
28+
wheeldir=$(dirname $WHEELNAME)
29+
echo $wheeldir
30+
31+
# delvewheel requires DLLs created by mingw64 to be stripped. This strips the
32+
# DLLs for GMP etc that will have been build previously.
33+
strip .local/bin/*.dll .local/lib/*.dll
34+
35+
# Make sure to leave the wheel in the same directory
36+
wheeldir=$(dirname $WHEELNAME)
37+
pushd $wheeldir
38+
# Unpack the wheel and strip any .pyd DLLs inside
39+
wheel unpack $WHEELNAME
40+
rm $WHEELNAME
41+
strip python_flint-*/flint/*.pyd
42+
wheel pack python_flint-*
43+
popd
44+
45+
# Make the wheel relocatable. This will fail with an error message about
46+
# --no-mangle if strip has not been applied to all mingw64-created .dll and
47+
# .pyd files that are needed for the wheel.
48+
delvewheel repair $WHEELNAME \
49+
-w $WHEELHOUSE \
50+
--add-path .local/bin:.local/lib/

‎setup.py

+22-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import sys
22
import os
3+
from subprocess import check_call
34

45
from distutils.core import setup
56
from distutils.extension import Extension
@@ -9,7 +10,27 @@
910
from distutils.sysconfig import get_config_vars
1011

1112
if sys.platform == 'win32':
12-
libraries = ["arb", "flint", "mpir", "mpfr", "pthreads"]
13+
#
14+
# This is used in CI to build wheels with mingw64
15+
#
16+
if os.getenv('PYTHON_FLINT_MINGW64'):
17+
libraries = ["arb", "flint", "mpfr", "gmp"]
18+
includedir = os.path.join(os.path.dirname(__file__), '.local', 'include')
19+
librarydir1 = os.path.join(os.path.dirname(__file__), '.local', 'bin')
20+
librarydir2 = os.path.join(os.path.dirname(__file__), '.local', 'lib')
21+
librarydirs = [librarydir1, librarydir2]
22+
default_include_dirs += [includedir]
23+
default_lib_dirs += librarydirs
24+
# Add gcc to the PATH in GitHub Actions when this setup.py is called by
25+
# cibuildwheel.
26+
os.environ['PATH'] += r';C:\msys64\mingw64\bin'
27+
elif os.getenv('PYTHON_FLINT_MINGW64_TMP'):
28+
# This would be used to build under Windows against these libraries if
29+
# they have been installed somewhere other than .local
30+
libraries = ["arb", "flint", "mpfr", "gmp"]
31+
else:
32+
# For the MSVC toolchain link with mpir instead of gmp
33+
libraries = ["arb", "flint", "mpir", "mpfr", "pthreads"]
1334
else:
1435
libraries = ["arb", "flint"]
1536
(opt,) = get_config_vars('OPT')

0 commit comments

Comments
 (0)