Skip to content

Commit fa78256

Browse files
kevinjqliukevinjqliu
authored
automate building Release Candidate in github action (#1391)
Related to #1306 and #872 Devlist discussion: https://lists.apache.org/thread/oowhcfwv3fcjzdzm76tbn99k5q84mr75 This PR changes `.github/workflows/python-release.yml` to build the artifacts for both svn and pypi and update the "how to release doc" to match the new process.
1 parent 29c9a96 commit fa78256

File tree

5 files changed

+253
-80
lines changed

5 files changed

+253
-80
lines changed

‎.github/workflows/python-release.yml

Lines changed: 183 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,118 @@
1717
# under the License.
1818
#
1919

20-
name: "Python Release"
20+
name: "Python Build Release Candidate"
2121

2222
on:
23+
push:
24+
tags:
25+
# Trigger this workflow when tag follows the versioning format: pyiceberg-<major>.<minor>.<patch>rc<release_candidate>
26+
# Example valid tags: pyiceberg-0.8.0rc2, pyiceberg-1.0.0rc1
27+
- 'pyiceberg-[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
2328
workflow_dispatch:
2429
inputs:
2530
version:
26-
description: 'Version'
31+
description: 'Version (e.g., 0.8.0)'
2732
type: string
28-
default: 'main'
29-
33+
required: true
34+
rc:
35+
description: 'Release Candidate (RC) (e.g., 1)'
36+
type: number
37+
required: true
3038

3139
jobs:
32-
build_wheels:
33-
name: Build wheels on ${{ matrix.os }}
40+
validate-inputs:
41+
runs-on: ubuntu-latest
42+
outputs:
43+
VERSION: ${{ steps.validate-inputs.outputs.VERSION }} # e.g. 0.8.0
44+
RC: ${{ steps.validate-inputs.outputs.RC }} # e.g. 1
45+
steps:
46+
- name: Validate and Extract Version and RC
47+
id: validate-inputs
48+
run: |
49+
if [ "$GITHUB_EVENT_NAME" = "push" ]; then
50+
echo "Workflow triggered by tag push."
51+
TAG=${GITHUB_REF#refs/tags/} # Extract the tag name
52+
VERSION_AND_RC=${TAG#pyiceberg-} # Remove the 'pyiceberg-' prefix
53+
VERSION=${VERSION_AND_RC%rc*} # Extract VERSION by removing everything after 'rc'
54+
RC=${VERSION_AND_RC#*rc} # Extract RC by keeping everything after 'rc'
55+
56+
if [[ -z "$VERSION" || -z "$RC" ]]; then
57+
echo "Error: Unable to parse VERSION or RC from tag ($TAG). Ensure the tag format is correct."
58+
exit 1
59+
fi
60+
else
61+
echo "Workflow triggered manually via workflow_dispatch."
62+
VERSION="${{ github.event.inputs.version }}"
63+
RC="${{ github.event.inputs.rc }}"
64+
65+
# Validate version (e.g., 1.0.0)
66+
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
67+
echo "Error: version ($VERSION) must be in the format: <number>.<number>.<number>"
68+
exit 1
69+
fi
70+
71+
# Validate rc (e.g., 1)
72+
if [[ ! "$RC" =~ ^[0-9]+$ ]]; then
73+
echo "Error: rc ($RC) must be in the format: <number>"
74+
exit 1
75+
fi
76+
fi
77+
78+
# Export variables for future steps
79+
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
80+
echo "RC=$RC" >> $GITHUB_OUTPUT
81+
82+
- name: Display Extracted Version and RC
83+
run: |
84+
echo "Using Version: ${{ steps.validate-inputs.outputs.VERSION }}"
85+
echo "Using RC: ${{ steps.validate-inputs.outputs.RC }}"
86+
87+
validate-library-version:
88+
runs-on: ubuntu-latest
89+
needs:
90+
- validate-inputs
91+
steps:
92+
- uses: actions/checkout@v4
93+
with:
94+
fetch-depth: 1
95+
96+
- uses: actions/setup-python@v5
97+
with:
98+
python-version: 3.12
99+
100+
- name: Install Poetry
101+
run: make install-poetry
102+
103+
- name: Validate current pyiceberg version
104+
env:
105+
VERSION: ${{ needs.validate-inputs.outputs.VERSION }}
106+
run: |
107+
# Extract the current version from Poetry
108+
current_pyiceberg_version=$(poetry version --short)
109+
echo "Detected Poetry version: $current_pyiceberg_version"
110+
111+
# Compare the input version with the Poetry version
112+
if [[ "$VERSION" != "$current_pyiceberg_version" ]]; then
113+
echo "Error: Input version ($VERSION) does not match the Poetry version ($current_pyiceberg_version)"
114+
exit 1
115+
fi
116+
117+
# SVN
118+
svn-build-artifacts:
119+
name: Build artifacts for SVN on ${{ matrix.os }}
34120
runs-on: ${{ matrix.os }}
121+
needs:
122+
- validate-inputs
123+
- validate-library-version
35124
strategy:
36125
matrix:
37126
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
38127

39128
steps:
40129
- uses: actions/checkout@v4
41130
with:
42-
fetch-depth: 0
131+
fetch-depth: 1
43132

44133
- uses: actions/setup-python@v5
45134
with:
@@ -50,11 +139,86 @@ jobs:
50139
3.12
51140
52141
- name: Install poetry
53-
run: pip install poetry
142+
run: make install-poetry
54143

55-
- name: Set version
56-
run: python -m poetry version "${{ inputs.version }}"
57-
if: "${{ github.event.inputs.version != 'main' }}"
144+
# Publish the source distribution with the version that's in
145+
# the repository, otherwise the tests will fail
146+
- name: Compile source distribution
147+
run: python3 -m poetry build --format=sdist
148+
if: startsWith(matrix.os, 'ubuntu')
149+
150+
- name: Build wheels
151+
uses: pypa/[email protected]
152+
with:
153+
output-dir: wheelhouse
154+
config-file: "pyproject.toml"
155+
env:
156+
# Ignore 32 bit architectures
157+
CIBW_ARCHS: "auto64"
158+
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
159+
CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
160+
CIBW_TEST_EXTRAS: "s3fs,glue"
161+
CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
162+
# There is an upstream issue with installing on MacOSX
163+
# https://github.com/pypa/cibuildwheel/issues/1603
164+
# Ignore tests for pypy since not all dependencies are compiled for it
165+
# and would require a local rust build chain
166+
CIBW_TEST_SKIP: "pp* *macosx*"
167+
168+
- name: Add source distribution
169+
if: startsWith(matrix.os, 'ubuntu')
170+
run: ls -lah dist/* && cp dist/* wheelhouse/
171+
172+
- uses: actions/upload-artifact@v4
173+
with:
174+
name: "svn-release-candidate-${{ matrix.os }}"
175+
path: ./wheelhouse/*
176+
177+
svn-merge-artifacts:
178+
runs-on: ubuntu-latest
179+
needs:
180+
- validate-inputs
181+
- svn-build-artifacts
182+
steps:
183+
- name: Merge Artifacts
184+
uses: actions/upload-artifact/merge@v4
185+
with:
186+
name: "svn-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
187+
pattern: svn-release-candidate*
188+
delete-merged: true
189+
190+
# PyPi
191+
pypi-build-artifacts:
192+
name: Build artifacts for PyPi on ${{ matrix.os }}
193+
runs-on: ${{ matrix.os }}
194+
needs:
195+
- validate-inputs
196+
- validate-library-version
197+
strategy:
198+
matrix:
199+
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
200+
201+
steps:
202+
- uses: actions/checkout@v4
203+
with:
204+
fetch-depth: 1
205+
206+
- uses: actions/setup-python@v5
207+
with:
208+
python-version: |
209+
3.9
210+
3.10
211+
3.11
212+
3.12
213+
214+
- name: Install poetry
215+
run: make install-poetry
216+
217+
- name: Set version with RC
218+
env:
219+
VERSION: ${{ needs.validate-inputs.outputs.VERSION }}
220+
RC: ${{ needs.validate-inputs.outputs.RC }}
221+
run: python -m poetry version "${{ env.VERSION }}rc${{ env.RC }}" # e.g., 0.8.0rc1
58222

59223
# Publish the source distribution with the version that's in
60224
# the repository, otherwise the tests will fail
@@ -86,15 +250,18 @@ jobs:
86250

87251
- uses: actions/upload-artifact@v4
88252
with:
89-
name: "release-${{ matrix.os }}"
253+
name: "pypi-release-candidate-${{ matrix.os }}"
90254
path: ./wheelhouse/*
91-
merge:
255+
256+
pypi-merge-artifacts:
92257
runs-on: ubuntu-latest
93-
needs: build_wheels
258+
needs:
259+
- validate-inputs
260+
- pypi-build-artifacts
94261
steps:
95262
- name: Merge Artifacts
96263
uses: actions/upload-artifact/merge@v4
97264
with:
98-
name: "release-${{ github.event.inputs.version }}"
99-
pattern: release-*
265+
name: "pypi-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
266+
pattern: pypi-release-candidate*
100267
delete-merged: true

‎mkdocs/docs/api.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1496,7 +1496,7 @@ print(ray_dataset.take(2))
14961496

14971497
### Daft
14981498

1499-
PyIceberg interfaces closely with Daft Dataframes (see also: [Daft integration with Iceberg](https://www.getdaft.io/projects/docs/en/latest/user_guide/integrations/iceberg.html)) which provides a full lazily optimized query engine interface on top of PyIceberg tables.
1499+
PyIceberg interfaces closely with Daft Dataframes (see also: [Daft integration with Iceberg](https://www.getdaft.io/projects/docs/en/stable/integrations/iceberg/)) which provides a full lazily optimized query engine interface on top of PyIceberg tables.
15001500

15011501
<!-- prettier-ignore-start -->
15021502

‎mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png

-216 KB
Binary file not shown.

‎mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png

-214 KB
Binary file not shown.

0 commit comments

Comments
 (0)