Skip to content

Commit 1cdc585

Browse files
kevinjqliukevinjqliu
authored
Infra: refactor release GitHub workflow (#1600)
Moved `svn-build-artifacts` and `svn-merge-artifacts` into `.github/workflows/svn-build-artifacts.yml` Moved `pypi-build-artifacts` and `pypi-merge-artifacts` into `.github/workflows/pypi-build-artifacts.yml` Test run: https://github.com/kevinjqliu/iceberg-python/actions/runs/13102665689 Still generates the same artifacts * `svn-release-candidate-${VERSION}rc${RC}` for SVN * `pypi-release-candidate-${VERSION}rc${RC}` for PyPi This is so can run pypi artifacts nightly.
1 parent fa78256 commit 1cdc585

File tree

3 files changed

+212
-140
lines changed

3 files changed

+212
-140
lines changed

‎.github/workflows/pypi-build-artifacts.yml

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: "Build PyPI Artifacts"
21+
22+
on:
23+
workflow_call:
24+
inputs:
25+
VERSION:
26+
required: true
27+
type: string
28+
RC:
29+
required: true
30+
type: string
31+
32+
jobs:
33+
pypi-build-artifacts:
34+
name: Build artifacts for PyPi on ${{ matrix.os }}
35+
runs-on: ${{ matrix.os }}
36+
strategy:
37+
matrix:
38+
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
39+
40+
steps:
41+
- uses: actions/checkout@v4
42+
with:
43+
fetch-depth: 1
44+
45+
- uses: actions/setup-python@v5
46+
with:
47+
python-version: |
48+
3.9
49+
3.10
50+
3.11
51+
3.12
52+
53+
- name: Install poetry
54+
run: make install-poetry
55+
56+
- name: Set version with RC
57+
env:
58+
VERSION: ${{ inputs.VERSION }}
59+
RC: ${{ inputs.RC }}
60+
run: python -m poetry version "${{ env.VERSION }}rc${{ env.RC }}" # e.g., 0.8.0rc1
61+
62+
# Publish the source distribution with the version that's in
63+
# the repository, otherwise the tests will fail
64+
- name: Compile source distribution
65+
run: python3 -m poetry build --format=sdist
66+
if: startsWith(matrix.os, 'ubuntu')
67+
68+
- name: Build wheels
69+
uses: pypa/[email protected]
70+
with:
71+
output-dir: wheelhouse
72+
config-file: "pyproject.toml"
73+
env:
74+
# Ignore 32 bit architectures
75+
CIBW_ARCHS: "auto64"
76+
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
77+
CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
78+
CIBW_TEST_EXTRAS: "s3fs,glue"
79+
CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
80+
# There is an upstream issue with installing on MacOSX
81+
# https://github.com/pypa/cibuildwheel/issues/1603
82+
# Ignore tests for pypy since not all dependencies are compiled for it
83+
# and would require a local rust build chain
84+
CIBW_TEST_SKIP: "pp* *macosx*"
85+
86+
- name: Add source distribution
87+
if: startsWith(matrix.os, 'ubuntu')
88+
run: ls -lah dist/* && cp dist/* wheelhouse/
89+
90+
- uses: actions/upload-artifact@v4
91+
with:
92+
name: "pypi-release-candidate-${{ matrix.os }}"
93+
path: ./wheelhouse/*
94+
95+
pypi-merge-artifacts:
96+
runs-on: ubuntu-latest
97+
needs:
98+
- pypi-build-artifacts
99+
steps:
100+
- name: Merge Artifacts
101+
uses: actions/upload-artifact/merge@v4
102+
with:
103+
name: "pypi-release-candidate-${{ inputs.VERSION }}rc${{ inputs.RC }}"
104+
pattern: pypi-release-candidate*
105+
delete-merged: true

‎.github/workflows/python-release.yml

Lines changed: 8 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -116,152 +116,20 @@ jobs:
116116
117117
# SVN
118118
svn-build-artifacts:
119-
name: Build artifacts for SVN on ${{ matrix.os }}
120-
runs-on: ${{ matrix.os }}
121119
needs:
122120
- validate-inputs
123121
- validate-library-version
124-
strategy:
125-
matrix:
126-
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
127-
128-
steps:
129-
- uses: actions/checkout@v4
130-
with:
131-
fetch-depth: 1
132-
133-
- uses: actions/setup-python@v5
134-
with:
135-
python-version: |
136-
3.9
137-
3.10
138-
3.11
139-
3.12
140-
141-
- name: Install poetry
142-
run: make install-poetry
143-
144-
# Publish the source distribution with the version that's in
145-
# the repository, otherwise the tests will fail
146-
- name: Compile source distribution
147-
run: python3 -m poetry build --format=sdist
148-
if: startsWith(matrix.os, 'ubuntu')
149-
150-
- name: Build wheels
151-
uses: pypa/[email protected]
152-
with:
153-
output-dir: wheelhouse
154-
config-file: "pyproject.toml"
155-
env:
156-
# Ignore 32 bit architectures
157-
CIBW_ARCHS: "auto64"
158-
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
159-
CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
160-
CIBW_TEST_EXTRAS: "s3fs,glue"
161-
CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
162-
# There is an upstream issue with installing on MacOSX
163-
# https://github.com/pypa/cibuildwheel/issues/1603
164-
# Ignore tests for pypy since not all dependencies are compiled for it
165-
# and would require a local rust build chain
166-
CIBW_TEST_SKIP: "pp* *macosx*"
167-
168-
- name: Add source distribution
169-
if: startsWith(matrix.os, 'ubuntu')
170-
run: ls -lah dist/* && cp dist/* wheelhouse/
171-
172-
- uses: actions/upload-artifact@v4
173-
with:
174-
name: "svn-release-candidate-${{ matrix.os }}"
175-
path: ./wheelhouse/*
176-
177-
svn-merge-artifacts:
178-
runs-on: ubuntu-latest
179-
needs:
180-
- validate-inputs
181-
- svn-build-artifacts
182-
steps:
183-
- name: Merge Artifacts
184-
uses: actions/upload-artifact/merge@v4
185-
with:
186-
name: "svn-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
187-
pattern: svn-release-candidate*
188-
delete-merged: true
122+
uses: ./.github/workflows/svn-build-artifacts.yml
123+
with:
124+
version: ${{ needs.validate-inputs.outputs.VERSION }}
125+
rc: ${{ needs.validate-inputs.outputs.RC }}
189126

190127
# PyPi
191128
pypi-build-artifacts:
192-
name: Build artifacts for PyPi on ${{ matrix.os }}
193-
runs-on: ${{ matrix.os }}
194129
needs:
195130
- validate-inputs
196131
- validate-library-version
197-
strategy:
198-
matrix:
199-
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
200-
201-
steps:
202-
- uses: actions/checkout@v4
203-
with:
204-
fetch-depth: 1
205-
206-
- uses: actions/setup-python@v5
207-
with:
208-
python-version: |
209-
3.9
210-
3.10
211-
3.11
212-
3.12
213-
214-
- name: Install poetry
215-
run: make install-poetry
216-
217-
- name: Set version with RC
218-
env:
219-
VERSION: ${{ needs.validate-inputs.outputs.VERSION }}
220-
RC: ${{ needs.validate-inputs.outputs.RC }}
221-
run: python -m poetry version "${{ env.VERSION }}rc${{ env.RC }}" # e.g., 0.8.0rc1
222-
223-
# Publish the source distribution with the version that's in
224-
# the repository, otherwise the tests will fail
225-
- name: Compile source distribution
226-
run: python3 -m poetry build --format=sdist
227-
if: startsWith(matrix.os, 'ubuntu')
228-
229-
- name: Build wheels
230-
uses: pypa/[email protected]
231-
with:
232-
output-dir: wheelhouse
233-
config-file: "pyproject.toml"
234-
env:
235-
# Ignore 32 bit architectures
236-
CIBW_ARCHS: "auto64"
237-
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
238-
CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
239-
CIBW_TEST_EXTRAS: "s3fs,glue"
240-
CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
241-
# There is an upstream issue with installing on MacOSX
242-
# https://github.com/pypa/cibuildwheel/issues/1603
243-
# Ignore tests for pypy since not all dependencies are compiled for it
244-
# and would require a local rust build chain
245-
CIBW_TEST_SKIP: "pp* *macosx*"
246-
247-
- name: Add source distribution
248-
if: startsWith(matrix.os, 'ubuntu')
249-
run: ls -lah dist/* && cp dist/* wheelhouse/
250-
251-
- uses: actions/upload-artifact@v4
252-
with:
253-
name: "pypi-release-candidate-${{ matrix.os }}"
254-
path: ./wheelhouse/*
255-
256-
pypi-merge-artifacts:
257-
runs-on: ubuntu-latest
258-
needs:
259-
- validate-inputs
260-
- pypi-build-artifacts
261-
steps:
262-
- name: Merge Artifacts
263-
uses: actions/upload-artifact/merge@v4
264-
with:
265-
name: "pypi-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
266-
pattern: pypi-release-candidate*
267-
delete-merged: true
132+
uses: ./.github/workflows/pypi-build-artifacts.yml
133+
with:
134+
version: ${{ needs.validate-inputs.outputs.VERSION }}
135+
rc: ${{ needs.validate-inputs.outputs.RC }}

‎.github/workflows/svn-build-artifacts.yml

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: "Build SVN Artifacts"
21+
22+
on:
23+
workflow_call:
24+
inputs:
25+
VERSION:
26+
required: true
27+
type: string
28+
RC:
29+
required: true
30+
type: string
31+
32+
jobs:
33+
svn-build-artifacts:
34+
name: Build artifacts for SVN on ${{ matrix.os }}
35+
runs-on: ${{ matrix.os }}
36+
strategy:
37+
matrix:
38+
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
39+
40+
steps:
41+
- uses: actions/checkout@v4
42+
with:
43+
fetch-depth: 1
44+
45+
- uses: actions/setup-python@v5
46+
with:
47+
python-version: |
48+
3.9
49+
3.10
50+
3.11
51+
3.12
52+
53+
- name: Install poetry
54+
run: make install-poetry
55+
56+
# Publish the source distribution with the version that's in
57+
# the repository, otherwise the tests will fail
58+
- name: Compile source distribution
59+
run: python3 -m poetry build --format=sdist
60+
if: startsWith(matrix.os, 'ubuntu')
61+
62+
- name: Build wheels
63+
uses: pypa/[email protected]
64+
with:
65+
output-dir: wheelhouse
66+
config-file: "pyproject.toml"
67+
env:
68+
# Ignore 32 bit architectures
69+
CIBW_ARCHS: "auto64"
70+
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
71+
CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
72+
CIBW_TEST_EXTRAS: "s3fs,glue"
73+
CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
74+
# There is an upstream issue with installing on MacOSX
75+
# https://github.com/pypa/cibuildwheel/issues/1603
76+
# Ignore tests for pypy since not all dependencies are compiled for it
77+
# and would require a local rust build chain
78+
CIBW_TEST_SKIP: "pp* *macosx*"
79+
80+
- name: Add source distribution
81+
if: startsWith(matrix.os, 'ubuntu')
82+
run: ls -lah dist/* && cp dist/* wheelhouse/
83+
84+
- uses: actions/upload-artifact@v4
85+
with:
86+
name: "svn-release-candidate-${{ matrix.os }}"
87+
path: ./wheelhouse/*
88+
89+
svn-merge-artifacts:
90+
runs-on: ubuntu-latest
91+
needs:
92+
- svn-build-artifacts
93+
steps:
94+
- name: Merge Artifacts
95+
uses: actions/upload-artifact/merge@v4
96+
with:
97+
name: "svn-release-candidate-${{ inputs.VERSION }}rc${{ inputs.RC }}"
98+
pattern: svn-release-candidate*
99+
delete-merged: true

0 commit comments

Comments
 (0)