Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: overhaul workflows to publish packages to new archive infra #38

Merged
merged 9 commits into from
Dec 24, 2024
Merged

feat: overhaul workflows to publish packages to new archive infra #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cb8d6da
feat: enhance test installable action to use new archives
khos2ow Nov 21, 2024
b13314b
feat: enhance local build and test package build action
khos2ow Dec 14, 2024
22a22b4
feat: add new action to publish packages
khos2ow Dec 14, 2024
60be3e0
feat: add new action to rebuild package sources
khos2ow Dec 14, 2024
80b0d0f
feat: refactor build scripts to publish to new server
khos2ow Dec 14, 2024
ad4a55a
chore: use the new consolidated actions repo
khos2ow Dec 17, 2024
7ac2ce4
fix: show the content of package folder after publish
khos2ow Dec 17, 2024
691746c
fix: make sure Release file exists and not empty
khos2ow Dec 19, 2024
6b83419
fix: package build and publish paths in local-build script
khos2ow Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .github/scripts/ext-arch.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,18 @@ publish() {
echo "CHLOG:Published arch package of $PACKAGE_NAME in $STAGE $DISTRO $CODENAME $ARCH (dry-run)"
}

# Create repo dist file - for new repository
generate_reprepro_dist() {
echo "function called: ${FUNCNAME[0]}"
archive_setup_scripts() {
echo "function called: ${FUNCNAME[0]}"
}

archive_cleanup_scripts() {
echo "function called: ${FUNCNAME[0]}"
}

# Setup repo layout
setup() {
echo "function called: ${FUNCNAME[0]}"

source_setup_scripts
archive_setup_scripts
}
213 changes: 129 additions & 84 deletions .github/scripts/ext-debian.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ set -o errexit
# Update the changelog to specify the target distribution codename
update_changelog() {
set -x
cd "${PKG_BUILD_DIR:?}/$PACKAGE_NAME"
cd "${PKG_BUILD_PATH:?}/$PACKAGE_NAME"
version=$(dpkg-parsechangelog --show-field Version)
dch --force-distribution --distribution "$CODENAME" --newversion "${version}-1regolith-$CODENAME" "Automated Voulage release"

Expand All @@ -18,12 +18,13 @@ update_changelog() {

# Determine if the changelog has the correct distribution codename
dist_valid() {
cd "${PKG_BUILD_DIR:?}/$PACKAGE_NAME"
cd "${PKG_BUILD_PATH:?}/$PACKAGE_NAME"

TOP_CHANGELOG_LINE=$(head -n 1 debian/changelog)
CHANGELOG_DIST=$(echo "$TOP_CHANGELOG_LINE" | cut -d' ' -f3)

cd - >/dev/null 2>&1

# echo "Checking $CODENAME and $CHANGELOG_DIST"
if [[ "$CHANGELOG_DIST" == *"$CODENAME"* ]]; then
return 0
Expand All @@ -36,14 +37,42 @@ stage_source() {
pushd .

echo "Preparing source for $PACKAGE_NAME"
cd "$PKG_BUILD_DIR/$PACKAGE_NAME" || exit
cd "$PKG_BUILD_PATH/$PACKAGE_NAME" || exit
debian_package_name=$(dpkg-parsechangelog --show-field Source)
full_version=$(dpkg-parsechangelog --show-field Version)
debian_version="${full_version%-*}"
cd "$PKG_BUILD_DIR" || exit
cd "$PKG_BUILD_PATH" || exit

echo "Generating source tarball from git repo."
tar --force-local -c -z -v -f "${debian_package_name}_${debian_version}.orig.tar.gz" --exclude .git\* --exclude /debian "$PACKAGE_NAME"
tar --force-local -c -z -v -f "${debian_package_name}_${debian_version}.orig.tar.gz" --exclude .git\* --exclude debian "$PACKAGE_NAME"

if [ "$LOCAL_BUILD" == "false" ]; then
debian_package_name_indicator="${debian_package_name:0:1}"
if [ "${debian_package_name:0:3}" == "lib" ]; then
debian_package_name_indicator="${debian_package_name:0:4}"
fi

# try to download the .orig.tar.gz from existing archive, and check if they are identical or not
wget -O "${debian_package_name}_${debian_version}-existing.orig.tar.gz" "http://archive.regolith-desktop.com/$DISTRO/$SUITE/pool/main/${debian_package_name_indicator}/${debian_package_name}/${debian_package_name}_${debian_version}.orig.tar.gz" || true

if [ -s "${debian_package_name}_${debian_version}-existing.orig.tar.gz" ]; then
if ! diff <(tar -tvzf "${debian_package_name}_${debian_version}.orig.tar.gz" | awk '{printf "%10s %s\n",$3,$6}' | sort -k 2 | sed 's|\./||') <(tar -tvzf "${debian_package_name}_${debian_version}-existing.orig.tar.gz" | awk '{printf "%10s %s\n",$3,$6}' | sort -k 2 | sed 's|\./||') ; then
# existing .orig.tar.gz file is different that the one we just built
# keep the one we just built and override push it to the repository.
rm -f "${debian_package_name}_${debian_version}-existing.orig.tar.gz" || true

echo "SRCLOG:$DISTRO=$CODENAME=$SUITE=${debian_package_name:0:1}=${debian_package_name}=${debian_package_name}_${debian_version}=${debian_package_name}_${debian_version}.orig.tar.gz"
else
# both .orig.tar.gz files are identical!
# remove the one we just built and reuse the existign one.
rm -f "${debian_package_name}_${debian_version}.orig.tar.gz" || true
mv "${debian_package_name}_${debian_version}-existing.orig.tar.gz" "${debian_package_name}_${debian_version}.orig.tar.gz"
fi
else
# there's no existing .orig.tar.gz file! Clean up the empty downloaded file.
rm -f "${debian_package_name}_${debian_version}-existing.orig.tar.gz" || true
fi
fi

popd
}
Expand All @@ -53,10 +82,11 @@ build_src_package() {

pushd .
echo "Building source package $PACKAGE_NAME"
cd "$PKG_BUILD_DIR/$PACKAGE_NAME" || exit
cd "$PKG_BUILD_PATH/$PACKAGE_NAME" || exit

sanitize_git
sudo apt update

sudo apt update
sudo apt build-dep -y .
debuild -S -sa

Expand All @@ -68,108 +98,123 @@ build_bin_package() {

pushd .
echo "Building binary package $PACKAGE_NAME"
cd "$PKG_BUILD_DIR/$PACKAGE_NAME" || exit
cd "$PKG_BUILD_PATH/$PACKAGE_NAME" || exit

debuild -sa -b
popd
}

# Internal (private) function
source_pkg_exists() {
SRC_PKG_VERSION=$(reprepro --basedir "$PKG_REPO_PATH" list "$CODENAME" "$1" | cut -d' ' -f3)

SRC_PKG_BUILD_VERSION=$(echo $2 | cut -d'-' -f1)
SRC_PKG_REPO_VERSION=$(echo $SRC_PKG_VERSION | cut -d'-' -f1)

if [ "$SRC_PKG_REPO_VERSION" == "$SRC_PKG_BUILD_VERSION" ]; then
return 0
else
return 1
fi
}

publish() {
cd "${PKG_BUILD_DIR:?}/$PACKAGE_NAME"
cd "${PKG_BUILD_PATH:?}/$PACKAGE_NAME"
version=$(dpkg-parsechangelog --show-field Version)
debian_package_name=$(dpkg-parsechangelog --show-field Source)
cd "$PKG_BUILD_DIR"
cd "$PKG_BUILD_PATH"

DEB_SRC_PKG_PATH="$PKG_BUILD_DIR/${debian_package_name}_${version}_source.changes"
DEB_SRC_PKG_PATH="$(pwd)/${debian_package_name}_${version}_source.changes"

if [ ! -f "$DEB_SRC_PKG_PATH" ]; then
echo "Failed to find changes file."
fi

if source_pkg_exists "$debian_package_name" "$version"; then
echo "Ignoring source package, already exists in target repository"
allow_failing_bin_package="true"
else
echo "Ingesting source package $debian_package_name into $PKG_REPO_PATH"
reprepro --basedir "$PKG_REPO_PATH" include "$CODENAME" "$DEB_SRC_PKG_PATH"
allow_failing_bin_package="false"
short_version="${version%-*}"

mkdir -p $PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

question: can you explain what SUITE models? How is it different than stage?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SUITE is introduced to be able to support multiple Regolith version per release STAGE. Suite is, very loosely, the same as Stage with one big difference. Stage is release stage from the SDLC point of view, but Suite is from the point of view of the published debian package archives.

The corresponding values and mappings are as follow:

Stage Suite Component
experimental experimental main
unstable unstable main
testing testing main
release-current stable main
release-3_2 stable v3.2
release-3_1 stable v3.1

Or in other words, in the scope of deb config line:

deb [arch=amd64] http://archive.regolith-desktop.com/ubuntu/unstable noble main
          |                                          |      |        |     |
          |                                          |      |        |     * Component
          |                                          |      |        * Distro Codename
          |                                          |      * Regolith Suite
          |                                          * Distro Name
          * Architecture

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LMK if this is not the case: SUITE is a debian repo concept. We would not necessarily use or need it for other packaging systems.


cp "$(pwd)/${debian_package_name}_${version}.dsc" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"
cp "$(pwd)/${debian_package_name}_${short_version}.orig.tar.gz" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"

if [ -f "$(pwd)/${debian_package_name}_${version}.debian.tar.xz" ]; then
cp "$(pwd)/${debian_package_name}_${version}.debian.tar.xz" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"
fi
if [ -f "$(pwd)/${debian_package_name}_${version}.diff.gz" ]; then
cp "$(pwd)/${debian_package_name}_${version}.diff.gz" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"
fi

if [ "$LOCAL_BUILD" == "false" ] && [ "$SUITE" == "stable" ]; then
mkdir -p "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$COMPONENT"
cd "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$COMPONENT" >/dev/null 2>&1

ln "../$SUITE/${debian_package_name}_${version}.dsc" .
ln "../$SUITE/${debian_package_name}_${short_version}.orig.tar.gz" .

if [ -f "../$SUITE/${debian_package_name}_${version}.debian.tar.xz" ]; then
ln "../$SUITE/${debian_package_name}_${version}.debian.tar.xz" .
fi
if [ -f "../$SUITE/${debian_package_name}_${version}.diff.gz" ]; then
ln "../$SUITE/${debian_package_name}_${version}.diff.gz" .
fi

cd - >/dev/null 2>&1
fi

echo "Publishing source package $debian_package_name into $PKG_PUBLISH_PATH"
fi

DEB_CONTROL_FILE="$PKG_BUILD_DIR/$PACKAGE_NAME/debian/control"
DEB_CONTROL_FILE="$PKG_BUILD_PATH/$PACKAGE_NAME/debian/control"
ALL_ARCH="$ARCH,all"

for target_arch in $(echo $ALL_ARCH | sed "s/,/ /g"); do
cat "$DEB_CONTROL_FILE" | grep ^Package: | cut -d' ' -f2 | while read -r bin_pkg; do
DEB_BIN_PKG_PATH="$(pwd)/${bin_pkg}_${version}_${target_arch}.deb"
if [ -f "$DEB_BIN_PKG_PATH" ]; then
if [ "$allow_failing_bin_package" == "true" ]; then
# If the source package/version already exists, allow the bin package build to fail (already exists)
reprepro --basedir "$PKG_REPO_PATH" includedeb "$CODENAME" "$DEB_BIN_PKG_PATH" || true
echo "Ingested binary package $DEB_BIN_PKG_PATH into $PKG_REPO_PATH"
else
reprepro --basedir "$PKG_REPO_PATH" includedeb "$CODENAME" "$DEB_BIN_PKG_PATH"
echo "Ingested binary package $DEB_BIN_PKG_PATH into $PKG_REPO_PATH"
fi
echo "CHLOG:Published ${bin_pkg}_${version}_${target_arch}.deb in $STAGE $DISTRO $CODENAME $ARCH from $PKG_LINE"
else
echo "Package $bin_pkg does not exist for $target_arch"
fi
done
cat "$DEB_CONTROL_FILE" | grep ^Package: | cut -d' ' -f2 | while read -r bin_pkg; do
DEB_BIN_PKG_PATH="$(pwd)/${bin_pkg}_${version}_${target_arch}.deb"

if [ -f "$DEB_BIN_PKG_PATH" ]; then
mkdir -p $PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE
cp "$DEB_BIN_PKG_PATH" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"

if [ "$LOCAL_BUILD" == "false" ] && [ "$SUITE" == "stable" ]; then
mkdir -p "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$COMPONENT"
cd "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$COMPONENT" >/dev/null 2>&1
ln "../$SUITE/${bin_pkg}_${version}_${target_arch}.deb" .
cd - >/dev/null 2>&1
fi

echo "Publishing deb package(s) $bin_pkg into $PKG_PUBLISH_PATH"
echo "CHLOG:Published ${bin_pkg}_${version}_${target_arch}.deb in $DISTRO/$CODENAME/$STAGE from $PKG_LINE"
else
echo "Package $bin_pkg does not exist for $target_arch"
fi
done
done
}

archive_setup_scripts() {
# Following allows for internal dependencies
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

question: can you elaborate on this? What would qualify as an internal dependency?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This part is equivalent of this which I moved from action file to here.

Not entirely sure if it is actually needed or not, but I think if a package that's being built relies on other packages that are maintained by Regolith (and basically only exists in archive.regolith-desktop.com and not Ubuntu or Debian public repos)

Copy link
Member

@kgilmer kgilmer Dec 17, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's actually needed 😄 . Presently we have one such case: sway-regolith depends on trawldb. The current approach is to break a build into two phases, for stages that doesn't already have some version of trawldb in the repo (for example, a new OS-codename being added) which result in commits like this and then a revert. Would be really nice to not have to deal with that sort of thing.

Does this mean that, for say an example where 3 packages need to be built, that each build runs in isolation (so that the dependencies are available), or something else? To contrast, the existing build system is kind of "one shot" meaning that any failure fails the whole build, and any dependency to a package build needs to already be available before the build starts. So, it's not possible to say "first build trawldb, then build sway-regolith" because, while it's possible to have trawldb build first, it is not ingested into the package repo until the end of the package build.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is a great point! I've also thinking about this and I think I might have a way forward to decentralize the process of building and publishing packages. If that works, then dependency management will ultimately boil down to "run the GitHub Action of the dependant package first and then run the Action of the other package."

We can track this further in #45 separately after getting this merged and get the ball rolling for unified archives in new server.

rm /tmp/Release || true
wget -P /tmp "http://archive.regolith-desktop.com/$DISTRO/$SUITE/dists/$CODENAME/Release" || true

if [ -s /tmp/Release ]; then
rm /tmp/Release

# Preliminary launchpad.net integration. Ideally gated from action rather than distro/stage
# Only packages for Ubuntu distro should go to launchpad
# Arch is set to amd64 because launchpad handles cross arch builds internally
# Remove STAGE check once testing and release PPAs are ready
# if [[ "$DISTRO" == "ubuntu" && "$ARCH" == "amd64" && "$STAGE" == "unstable" ]]; then
# LAUNCHPAD_REPO="ppa:regolith-desktop/$CODENAME-$STAGE"
#
# dput -f $LAUNCHPAD_REPO $DEB_SRC_PKG_PATH || true
#
# echo "CHLOG: Published $PACKAGE_NAME to $LAUNCHPAD_REPO"
# fi
local repo_line=""
if [ "$LOCAL_BUILD" == "false" ] && [ "$SUITE" == "stable" ]; then
# fixed version component
repo_line="http://archive.regolith-desktop.com/$DISTRO/$SUITE $CODENAME v$COMPONENT"
else
# main component
repo_line="http://archive.regolith-desktop.com/$DISTRO/$SUITE $CODENAME $COMPONENT"
fi

echo "Adding repo to apt: $repo_line"
wget -qO - http://archive.regolith-desktop.com/regolith.key | sudo apt-key add -
echo "deb [arch=$ARCH] $repo_line" | sudo tee /etc/apt/sources.list.d/regolith.list

sudo apt update
fi

if [ -f "/etc/apt/sources.list.d/regolith-local.list" ]; then
sudo rm /etc/apt/sources.list.d/regolith-local.list
echo "Cleaned up temp apt repo"
fi
}

# Create repo dist file
generate_reprepro_dist() {
echo "Origin: $PACKAGE_REPO_URL" > "$PKG_REPO_PATH/conf/distributions"
echo "Label: $PACKAGE_REPO_URL" >> "$PKG_REPO_PATH/conf/distributions"
echo "Codename: $CODENAME" >> "$PKG_REPO_PATH/conf/distributions"
echo "Architectures: $ARCH source" >> "$PKG_REPO_PATH/conf/distributions"
echo "Components: main" >> "$PKG_REPO_PATH/conf/distributions"
echo "Description: $STAGE $DISTRO $CODENAME $ARCH" >> "$PKG_REPO_PATH/conf/distributions"
echo "SignWith: $APT_KEY" >> "$PKG_REPO_PATH/conf/distributions"
archive_cleanup_scripts() {
# Remove regolith repo from build system apt config
if [ -f /etc/apt/sources.list.d/regolith.list ]; then
sudo rm -f /etc/apt/sources.list.d/regolith.list
fi
}

# Setup debian repo
setup() {
if [ ! -d "$PKG_REPO_PATH/conf" ]; then
echo "Creating conf dir"
mkdir -p "$PKG_REPO_PATH/conf"
fi

if [ ! -f "$PKG_REPO_PATH/conf/distributions" ]; then
echo "Package metadata not found, creating conf dir"
generate_reprepro_dist
cat "$PKG_REPO_PATH/conf/distributions"
else
echo "Existing metadata:"
cat "$PKG_REPO_PATH/conf/distributions"
fi

source_setup_scripts
archive_setup_scripts
}
19 changes: 10 additions & 9 deletions .github/scripts/ext-git.sh
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,25 @@ checkout() {
set -e

if [ -z "$PACKAGE_URL" ]; then
echo "Package model is invalid. Model field 'source' undefined, aborting."
echo "Error: package model is invalid. Model field 'source' undefined, aborting."
exit 1
fi

if [ -d "$PKG_BUILD_DIR/$PACKAGE_NAME" ]; then
if [ -d "$PKG_BUILD_PATH/$PACKAGE_NAME" ]; then
echo "Deleting existing repo, $PACKAGE_NAME"
rm -Rfv "${PKG_BUILD_DIR:?}/$PACKAGE_NAME"
rm -Rfv "${PKG_BUILD_PATH:?}/$PACKAGE_NAME"
fi

if [ ! -d "$PKG_BUILD_DIR" ]; then
echo "Creating build directory $PKG_BUILD_DIR"
mkdir -p "$PKG_BUILD_DIR" || {
echo "Failed to create build dir $PKG_BUILD_DIR, aborting."
if [ ! -d "$PKG_BUILD_PATH" ]; then
echo "Creating build directory $PKG_BUILD_PATH"

mkdir -p "$PKG_BUILD_PATH" || {
echo "Error: failed to create build dir $PKG_BUILD_PATH, aborting."
exit 1
}
fi

cd "$PKG_BUILD_DIR" || exit
cd "$PKG_BUILD_PATH" || exit
git clone --recursive "$PACKAGE_URL" -b "$PACKAGE_REF" "$PACKAGE_NAME"

cd - >/dev/null 2>&1 || exit
Expand All @@ -36,4 +37,4 @@ sanitize_git() {
rm -Rf .git
echo "Removed $(pwd).git directory before building to appease debuild."
fi
}
}
10 changes: 10 additions & 0 deletions .github/scripts/ext-rpm.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,16 @@ publish() {
:
}

archive_setup_scripts() {
# Setting up local archive repo for internal dependencies
:
}

archive_cleanup_scripts() {
# Cleaning up local archive repo definition
:
}

# Setup debian repo
setup() {
:
Expand Down
10 changes: 7 additions & 3 deletions .github/scripts/ext-template.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,18 @@ publish() {
echo todo
}

# Create repo dist file - for new repository
generate_reprepro_dist() {
echo todo
archive_setup_scripts() {
echo todo
}

archive_cleanup_scripts() {
echo todo
}

# Setup repo layout
setup() {
echo todo

source_setup_scripts
archive_setup_scripts
}
Loading