Build unstable Packages debian testing #39
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Package Builder Debian v5 | |
run-name: Build ${{ inputs.stage }} Packages ${{ inputs.distro }} ${{ inputs.codename }} ${{ inputs.arch }} | |
on: | |
workflow_dispatch: | |
inputs: | |
do-publish: | |
description: "Publish after build" | |
type: choice | |
options: | |
- "yes" | |
- "no" | |
required: true | |
default: "yes" | |
force-build: | |
description: "Force build without check for changes" | |
type: choice | |
options: | |
- "yes" | |
- "no" | |
required: true | |
default: "no" | |
stage: | |
description: "Stage to build" | |
type: choice | |
options: | |
- all | |
- experimental | |
- unstable | |
- testing | |
- release-3_0 | |
- release-3_1 | |
- release-3_2 | |
required: true | |
default: "unstable" | |
distro: | |
description: "Distro to build (debian, ubuntu)" | |
type: choice | |
options: | |
- "" | |
- debian | |
- ubuntu | |
required: false | |
default: "" | |
codename: | |
description: "Codename to build (e.g. noble, bookworm)" | |
type: string | |
required: false | |
default: "" | |
arch: | |
description: "Architecture to build (amd64, arm64)" | |
type: choice | |
options: | |
- "" | |
- amd64 | |
- arm64 | |
required: false | |
default: "" | |
workflow_call: | |
inputs: | |
do-publish: | |
description: "Publish after build" | |
type: string | |
required: true | |
default: "yes" | |
force-build: | |
description: "Force build without check for changes" | |
type: string | |
required: true | |
default: "no" | |
stage: | |
description: "Stage to build" | |
type: string | |
required: true | |
default: "unstable" | |
distro: | |
description: "Distro to build (debian, ubuntu)" | |
type: string | |
required: false | |
codename: | |
description: "Codename to build (e.g. noble, bookworm)" | |
type: string | |
required: false | |
arch: | |
description: "Architecture to build (amd64, arm64)" | |
type: string | |
required: false | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: true | |
# Need to explicitly set shell according to: https://github.com/actions/runner/issues/353#issuecomment-1067227665 | |
defaults: | |
run: | |
shell: bash | |
jobs: | |
matrix-builder: | |
runs-on: ubuntu-24.04 | |
container: "ghcr.io/regolith-linux/ci-ubuntu:noble-amd64" | |
outputs: | |
stages: ${{ steps.calc-matrix.outputs.stages }} | |
distros: ${{ steps.calc-matrix.outputs.distros }} | |
codenames: ${{ steps.calc-matrix.outputs.codenames }} | |
arches: ${{ steps.calc-matrix.outputs.arches }} | |
suites: ${{ steps.calc-matrix.outputs.suites }} | |
runners: ${{ steps.calc-matrix.outputs.runners }} | |
includes: ${{ steps.calc-matrix.outputs.includes }} | |
excludes: ${{ steps.calc-matrix.outputs.excludes }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Calculate Matrix | |
id: calc-matrix | |
run: | | |
if [ -n "${{ inputs.stage }}" ] && [ "${{ inputs.stage }}" != "all" ]; then | |
STAGES=(${{ inputs.stage }}) | |
else | |
STAGES=(experimental testing unstable) | |
fi | |
DISTROS=() | |
CODENAMES=() | |
if [ -n "${{ inputs.arch }}" ]; then | |
ARCHES=(${{ inputs.arch }}) | |
else | |
ARCHES=(amd64 arm64) | |
fi | |
INCLUDES=() | |
EXCLUDES=() | |
valid_distro_codenames=() | |
for stage in "${STAGES[@]}"; do | |
if [ ! -d "stage/${stage}" ]; then | |
echo "Package model for stage ${stage} not found!" | |
continue | |
fi | |
for dir in $(find stage/${stage}/ -mindepth 2 -maxdepth 2 -type d | sort); do | |
for arch in "${ARCHES[@]}"; do | |
distro=$(echo $dir | cut -d/ -f3) | |
if [ -n "${{ inputs.distro }}" ] && [ "${{ inputs.distro }}" != "$distro" ]; then | |
continue | |
fi | |
codename=$(echo $dir | cut -d/ -f4) | |
if [ -n "${{ inputs.codename }}" ] && [ "${{ inputs.codename }}" != "$codename" ]; then | |
continue | |
fi | |
if [[ ! " ${DISTROS[*]} " =~ [[:space:]]${distro}[[:space:]] ]]; then | |
DISTROS+=("${distro}") | |
fi | |
if [[ ! " ${CODENAMES[*]} " =~ [[:space:]]${codename}[[:space:]] ]]; then | |
CODENAMES+=("${codename}") | |
fi | |
if [[ ! " ${valid_distro_codenames[*]} " =~ [[:space:]]${distro}-${codename}[[:space:]] ]]; then | |
valid_distro_codenames+=("${distro}-${codename}") | |
fi | |
done | |
done | |
done | |
for distro in "${DISTROS[@]}"; do | |
for codename in "${CODENAMES[@]}"; do | |
if [[ ! " ${valid_distro_codenames[*]} " =~ [[:space:]]${distro}-${codename}[[:space:]] ]]; then | |
EXCLUDES+=($(jq -n -c --arg distro "$distro" --arg codename "$codename" '$ARGS.named')) | |
fi | |
done | |
done | |
if [[ "${{ inputs.stage }}" == "release-"* ]]; then | |
EXCLUDES+=($(jq -n -c --arg distro "debian" --arg codename "testing" '$ARGS.named')) | |
fi | |
SUITES=$(jq -n "$(jq -n -c \ | |
--argjson experimental "$(jq -n -c --arg suite "experimental" --arg component "main" '$ARGS.named')" \ | |
--argjson unstable "$(jq -n -c --arg suite "unstable" --arg component "main" '$ARGS.named')" \ | |
--argjson testing "$(jq -n -c --arg suite "testing" --arg component "main" '$ARGS.named')" \ | |
--argjson stable "$(jq -n -c --arg suite "stable" --arg component "main" '$ARGS.named')" \ | |
--argjson release-3_0 "$(jq -n -c --arg suite "stable" --arg component "v3.0" '$ARGS.named')" \ | |
--argjson release-3_1 "$(jq -n -c --arg suite "stable" --arg component "v3.1" '$ARGS.named')" \ | |
--argjson release-3_2 "$(jq -n -c --arg suite "stable" --arg component "v3.2" '$ARGS.named')" \ | |
'$ARGS.named'\ | |
)" '$ARGS.named') | |
echo "stages=$(jq -n -c '$ARGS.positional' --args -- "${STAGES[@]}")" >> $GITHUB_OUTPUT | |
echo "distros=$(jq -n -c '$ARGS.positional' --args -- "${DISTROS[@]}")" >> $GITHUB_OUTPUT | |
echo "codenames=$(jq -n -c '$ARGS.positional' --args -- "${CODENAMES[@]}")" >> $GITHUB_OUTPUT | |
echo "arches=$(jq -n -c '$ARGS.positional' --args -- "${ARCHES[@]}")" >> $GITHUB_OUTPUT | |
echo "suites=$(jq -n -c "${SUITES}" '$ARGS.named')" >> $GITHUB_OUTPUT | |
echo "runners=$(jq -n -c "$(jq -n -c --arg amd64 "ubuntu-24.04" --arg arm64 "ubuntu-24.04-arm" '$ARGS.named')" '$ARGS.named')" >> $GITHUB_OUTPUT | |
echo "includes=$(jq -n -c "[$(printf '%s\n' "${INCLUDES[@]}" | paste -sd,)]" '$ARGS.named')" >> $GITHUB_OUTPUT | |
echo "excludes=$(jq -n -c "[$(printf '%s\n' "${EXCLUDES[@]}" | paste -sd,)]" '$ARGS.named')" >> $GITHUB_OUTPUT | |
# build packages and sources | |
build: | |
runs-on: ${{ fromJSON(needs.matrix-builder.outputs.runners)[matrix.arch] }} | |
needs: matrix-builder | |
container: "ghcr.io/regolith-linux/ci-${{ matrix.distro }}:${{ matrix.codename }}-${{ matrix.arch }}" | |
strategy: | |
fail-fast: false | |
matrix: | |
stage: ${{ fromJSON(needs.matrix-builder.outputs.stages) }} | |
distro: ${{ fromJSON(needs.matrix-builder.outputs.distros) }} | |
codename: ${{ fromJSON(needs.matrix-builder.outputs.codenames) }} | |
arch: ${{ fromJSON(needs.matrix-builder.outputs.arches) }} | |
include: ${{ fromJSON(needs.matrix-builder.outputs.includes) }} | |
exclude: ${{ fromJSON(needs.matrix-builder.outputs.excludes) }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Set Job Parameters | |
id: init | |
run: | | |
export GH_WORKSPACE="/__w/voulage/voulage" | |
echo "gh-repo-path=${GH_WORKSPACE}" >> $GITHUB_OUTPUT | |
echo "changelogs-path=${GH_WORKSPACE}/changelogs" >> $GITHUB_OUTPUT | |
echo "manifest-path=${GH_WORKSPACE}/manifests" >> $GITHUB_OUTPUT | |
echo "package-build-path=${GH_WORKSPACE}/packages" >> $GITHUB_OUTPUT | |
echo "package-publish-path=${GH_WORKSPACE}/publish" >> $GITHUB_OUTPUT | |
echo "stage=${{ matrix.stage }}" >> $GITHUB_OUTPUT | |
echo "distro=${{ matrix.distro }}" >> $GITHUB_OUTPUT | |
echo "codename=${{ matrix.codename }}" >> $GITHUB_OUTPUT | |
echo "arch=${{ matrix.arch }}" >> $GITHUB_OUTPUT | |
echo "suite=${{ fromJSON(needs.matrix-builder.outputs.suites)[matrix.stage]['suite'] }}" >> $GITHUB_OUTPUT | |
echo "component=${{ fromJSON(needs.matrix-builder.outputs.suites)[matrix.stage]['component'] }}" >> $GITHUB_OUTPUT | |
echo "target=${{ matrix.stage }}-${{ matrix.distro }}-${{ matrix.codename }}-${{ matrix.arch }}" >> $GITHUB_OUTPUT | |
- name: Environment Setup | |
run: | | |
set -e | |
mkdir -p ${{ steps.init.outputs.changelogs-path }} || true | |
mkdir -p ${{ steps.init.outputs.manifest-path }} || true | |
sudo rm -rf /etc/apt/sources.list.d/regolith.list | |
- name: Pull Manifest | |
if: inputs.force-build == 'no' | |
run: | | |
cp "${{ steps.init.outputs.manifest-path }}/${{ steps.init.outputs.distro }}/${{ steps.init.outputs.codename }}/${{ steps.init.outputs.suite }}-${{ steps.init.outputs.component }}/manifest.txt" ${{ steps.init.outputs.manifest-path }} || true | |
echo -e "\033[0;34mCurrent manifest:\033[0m" | |
cat ${{ steps.init.outputs.manifest-path }}/manifest.txt || true | |
- name: Check for changes | |
id: changes | |
run: | | |
set -e | |
set -o pipefail | |
if [ "${{ inputs.force-build }}" == "no" ]; then | |
CHANGE_OUTPUT=$(${{ steps.init.outputs.gh-repo-path }}/.github/scripts/main.sh \ | |
check \ | |
--extension ${{ steps.init.outputs.gh-repo-path }}/.github/scripts/ext-debian.sh \ | |
--git-repo-path ${{ steps.init.outputs.gh-repo-path }} \ | |
--manifest-path ${{ steps.init.outputs.manifest-path }} \ | |
--pkg-build-path ${{ steps.init.outputs.package-build-path }} \ | |
--pkg-publish-path ${{ steps.init.outputs.package-publish-path }} \ | |
--distro "${{ steps.init.outputs.distro }}" \ | |
--codename "${{ steps.init.outputs.codename }}" \ | |
--arch "${{ steps.init.outputs.arch }}" \ | |
--stage "${{ steps.init.outputs.stage }}" \ | |
--suite "${{ steps.init.outputs.suite }}" \ | |
--component "${{ steps.init.outputs.component }}" | tail -n1) | |
if [ "$CHANGE_OUTPUT" == "No package changes found, exiting." ]; then | |
echo "changed=0" >> $GITHUB_OUTPUT | |
echo -e "\033[0;31mNo package changes to build.\033[0m" | |
else | |
echo "changed=1" >> $GITHUB_OUTPUT | |
echo -e "\033[0;34mNew Manifest:\033[0m" | |
cat ${{ steps.init.outputs.manifest-path }}/next-manifest.txt | |
fi | |
else | |
echo "changed=1" >> $GITHUB_OUTPUT | |
echo -e "\033[0;34m'force-build' selected, proceeding with building package.\033[0m" | |
fi | |
- name: Import GPG Key | |
uses: regolith-linux/actions/import-gpg@main | |
if: steps.changes.outputs.changed == 1 | |
with: | |
gpg-key: "${{ secrets.PACKAGE_PRIVATE_KEY2 }}" | |
- name: Build Packages | |
if: steps.changes.outputs.changed == 1 | |
env: | |
DEBEMAIL: "[email protected]" | |
DEBFULLNAME: "Regolith Linux" | |
DEBIAN_FRONTEND: noninteractive | |
run: | | |
set -e | |
set -o pipefail | |
${{ steps.init.outputs.gh-repo-path }}/.github/scripts/main.sh \ | |
build \ | |
--extension ${{ steps.init.outputs.gh-repo-path }}/.github/scripts/ext-debian.sh \ | |
--git-repo-path ${{ steps.init.outputs.gh-repo-path }} \ | |
--manifest-path ${{ steps.init.outputs.manifest-path }} \ | |
--pkg-build-path ${{ steps.init.outputs.package-build-path }} \ | |
--pkg-publish-path ${{ steps.init.outputs.package-publish-path }} \ | |
--distro "${{ steps.init.outputs.distro }}" \ | |
--codename "${{ steps.init.outputs.codename }}" \ | |
--arch "${{ steps.init.outputs.arch }}" \ | |
--stage "${{ steps.init.outputs.stage }}" \ | |
--suite "${{ steps.init.outputs.suite }}" \ | |
--component "${{ steps.init.outputs.component }}" | tee -a ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.raw.txt | |
if [ -f ${{ steps.init.outputs.manifest-path }}/next-manifest.txt ]; then | |
echo -e "\033[0;31mTemp manifest not deleted by main.sh, build aborted/failed.\033[0m" | |
exit 1 | |
fi | |
echo "::group::Preparing manifest for upload" | |
mv ${{ steps.init.outputs.manifest-path }}/manifest.txt ${{ steps.init.outputs.manifest-path }}/${{ steps.init.outputs.distro }}_${{ steps.init.outputs.codename }}_${{ steps.init.outputs.suite }}-${{ steps.init.outputs.component }}_manifest.txt | |
echo "${{ steps.init.outputs.distro }}_${{ steps.init.outputs.codename }}_${{ steps.init.outputs.suite }}-${{ steps.init.outputs.component }}_manifest.txt prepared successfully." | |
echo "::endgroup::" | |
echo "::group::Preparing changelog and sourcelog for upload" | |
touch ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.txt | |
if grep "^CHLOG:" ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.raw.txt >/dev/null; then | |
grep "^CHLOG:" ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.raw.txt | cut -c 7- > ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.txt | |
fi | |
touch ${{ steps.init.outputs.changelogs-path }}/SOURCELOG_${{ steps.init.outputs.target }}.txt | |
if grep "^SRCLOG:" ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.raw.txt >/dev/null; then | |
grep "^SRCLOG:" ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.raw.txt | cut -c 8- > ${{ steps.init.outputs.changelogs-path }}/SOURCELOG_${{ steps.init.outputs.target }}.txt | |
fi | |
if [ ! -s ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.txt ]; then | |
rm ${{ steps.init.outputs.changelogs-path }}/CHANGELOG_${{ steps.init.outputs.target }}.txt | |
else | |
echo "CHANGELOG_${{ steps.init.outputs.target }}.txt prepared successfully." | |
fi | |
if [ ! -s ${{ steps.init.outputs.changelogs-path }}/SOURCELOG_${{ steps.init.outputs.target }}.txt ]; then | |
rm ${{ steps.init.outputs.changelogs-path }}/SOURCELOG_${{ steps.init.outputs.target }}.txt | |
else | |
echo "SOURCELOG_${{ steps.init.outputs.target }}.txt prepared successfully." | |
fi | |
echo "::endgroup::" | |
- name: Setup SSH | |
uses: regolith-linux/actions/setup-ssh@main | |
if: inputs.do-publish == 'yes' && steps.changes.outputs.changed == 1 | |
with: | |
ssh-host: "${{ secrets.KAMATERA_HOSTNAME2 }}" | |
ssh-key: "${{ secrets.KAMATERA_SSH_KEY }}" | |
- name: Upload Package | |
uses: regolith-linux/actions/upload-files@main | |
if: inputs.do-publish == 'yes' && steps.changes.outputs.changed == 1 | |
env: | |
server-address: "${{ secrets.KAMATERA_HOSTNAME2 }}" | |
server-username: "${{ secrets.KAMATERA_USERNAME }}" | |
with: | |
upload-from: "${{ steps.init.outputs.package-publish-path }}" | |
upload-pattern: "*" | |
upload-to-base: "/opt/archives/packages/" | |
upload-to-folder: "voulage/" | |
- name: Upload SourceLog | |
uses: regolith-linux/actions/upload-files@main | |
if: inputs.do-publish == 'yes' && steps.changes.outputs.changed == 1 | |
env: | |
server-address: "${{ secrets.KAMATERA_HOSTNAME2 }}" | |
server-username: "${{ secrets.KAMATERA_USERNAME }}" | |
with: | |
upload-from: "${{ steps.init.outputs.changelogs-path }}" | |
upload-pattern: "SOURCELOG_${{ steps.init.outputs.target }}.txt" | |
upload-to-base: "/opt/archives/workspace/" | |
upload-to-folder: "voulage/" | |
- name: Log Build Output | |
if: steps.changes.outputs.changed == 1 | |
run: | | |
cat ${{ steps.init.outputs.manifest-path }}/${{ steps.init.outputs.distro }}_${{ steps.init.outputs.codename }}_${{ steps.init.outputs.suite }}-${{ steps.init.outputs.component }}_manifest.txt || true | |
echo -e "\033[0;34mContent of package publish path:\033[0m" | |
find ${{ steps.init.outputs.package-publish-path }} | |
- name: Upload Manifests | |
uses: actions/upload-artifact@v4 | |
if: steps.changes.outputs.changed == 1 | |
with: | |
name: MANIFESTS_${{ steps.init.outputs.target }} | |
path: "${{ steps.init.outputs.manifest-path }}/*_manifest.txt" | |
- name: Upload Changelogs | |
uses: actions/upload-artifact@v4 | |
if: steps.changes.outputs.changed == 1 | |
with: | |
name: CHANGELOG_${{ steps.init.outputs.target }} | |
path: ${{ steps.init.outputs.changelogs-path }}/*${{ steps.init.outputs.target }}.txt | |
# calculate changelogs | |
changelogs: | |
runs-on: ubuntu-24.04 | |
container: "ghcr.io/regolith-linux/ci-ubuntu:noble-amd64" | |
needs: build | |
if: ${{ !failure() && !cancelled() && inputs.do-publish == 'yes' }} | |
outputs: | |
package-changed: ${{ steps.check.outputs.package-changed }} | |
source-changed: ${{ steps.check.outputs.source-changed }} | |
steps: | |
- name: Download Artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
path: changelogs | |
pattern: CHANGELOG_* | |
merge-multiple: true | |
- name: Check Changelogs | |
id: check | |
run: | | |
set -e | |
if [ ! -d changelogs ]; then | |
echo "No package file found to publish!" | |
echo "No source file found to rebuild!" | |
echo "package-changed=0" >> $GITHUB_OUTPUT | |
echo "source-changed=0" >> $GITHUB_OUTPUT | |
else | |
ls -R changelogs/ | |
echo "package-changed=$(find changelogs -name CHANGELOG_\*.txt | wc -l)" >> $GITHUB_OUTPUT | |
echo "source-changed=$(find changelogs -name SOURCELOG_\*.txt | wc -l)" >> $GITHUB_OUTPUT | |
fi | |
# rebuild sources | |
source: | |
needs: [changelogs] | |
if: ${{ !failure() && !cancelled() && inputs.do-publish == 'yes' && needs.changelogs.outputs.source-changed != 0 }} | |
uses: ./.github/workflows/rebuild-sources.yml | |
with: | |
pull-from: /opt/archives/workspace/ | |
push-to: /opt/archives/packages/ | |
secrets: inherit | |
# publish archives | |
publish: | |
needs: [changelogs, source] | |
if: ${{ !failure() && !cancelled() && inputs.do-publish == 'yes' && needs.changelogs.outputs.package-changed != 0 }} | |
uses: ./.github/workflows/publish-packages.yml | |
with: | |
packages-path: /opt/archives/packages/ | |
secrets: inherit | |
# update manifests | |
manifests: | |
runs-on: ubuntu-24.04 | |
container: "ghcr.io/regolith-linux/ci-ubuntu:noble-amd64" | |
needs: [changelogs, publish] | |
if: ${{ !failure() && !cancelled() && inputs.do-publish == 'yes' }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Download Artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
path: manifests | |
pattern: MANIFESTS_* | |
merge-multiple: true | |
- name: Check Manifests | |
run: | | |
for file in manifests/*_manifest.txt; do | |
if [ ! -f "$file" ]; then | |
continue | |
fi | |
filename="$(basename $file)" | |
distro="$(echo $filename | cut -d"_" -f1)" | |
codename="$(echo $filename | cut -d"_" -f2)" | |
suite="$(echo $filename | cut -d"_" -f3)" | |
name="$(echo $filename | cut -d"_" -f4)" | |
mkdir -p manifests/$distro/$codename/$suite/ | |
mv $file manifests/$distro/$codename/$suite/$name | |
done | |
- name: Push Manifest Changes | |
uses: stefanzweifel/git-auto-commit-action@v5 | |
env: | |
GITHUB_TOKEN: ${{ secrets.ORG_BROADCAST_TOKEN }} | |
with: | |
file_pattern: "*" | |
commit_message: "chore: update ${{ inputs.stage }} manifest for all packages" | |
commit_user_name: regolith-ci-bot | |
commit_user_email: [email protected] | |
commit_author: "regolith-ci-bot <[email protected]>" | |
# run the tests | |
test: | |
needs: [manifests] | |
if: ${{ !failure() && !cancelled() && inputs.do-publish == 'yes' }} | |
uses: ./.github/workflows/test-desktop-installable2.yml | |
with: | |
stage: ${{ inputs.stage }} | |
distro: ${{ inputs.distro }} | |
codename: ${{ inputs.codename }} | |
arch: ${{ inputs.arch }} |