Merge pull request #38 from regolith-linux/migrate

feat: overhaul workflows to publish packages to new archive infra
regolith-linux · Dec 24, 2024 · 422dda2 · 422dda2
2 parents 44748ad + 6b83419
commit 422dda2
Show file tree

Hide file tree

Showing 19 changed files with 1,791 additions and 528 deletions.
diff --git a/.github/scripts/ext-arch.sh b/.github/scripts/ext-arch.sh
@@ -57,14 +57,18 @@ publish() {
   echo "CHLOG:Published arch package of $PACKAGE_NAME in $STAGE $DISTRO $CODENAME $ARCH (dry-run)"
 }
 
-# Create repo dist file - for new repository
-generate_reprepro_dist() {
-    echo "function called: ${FUNCNAME[0]}"
+archive_setup_scripts() {
+  echo "function called: ${FUNCNAME[0]}"
+}
+
+archive_cleanup_scripts() {
+  echo "function called: ${FUNCNAME[0]}"
 }
 
 # Setup repo layout
 setup() {
   echo "function called: ${FUNCNAME[0]}"
 
   source_setup_scripts
+  archive_setup_scripts
 }
diff --git a/.github/scripts/ext-debian.sh b/.github/scripts/ext-debian.sh
@@ -9,7 +9,7 @@ set -o errexit
 # Update the changelog to specify the target distribution codename
 update_changelog() {
   set -x
-  cd "${PKG_BUILD_DIR:?}/$PACKAGE_NAME"
+  cd "${PKG_BUILD_PATH:?}/$PACKAGE_NAME"
   version=$(dpkg-parsechangelog --show-field Version)
   dch --force-distribution --distribution "$CODENAME" --newversion "${version}-1regolith-$CODENAME" "Automated Voulage release"
 
@@ -18,12 +18,13 @@ update_changelog() {
 
 # Determine if the changelog has the correct distribution codename
 dist_valid() {
-  cd "${PKG_BUILD_DIR:?}/$PACKAGE_NAME"
+  cd "${PKG_BUILD_PATH:?}/$PACKAGE_NAME"
 
   TOP_CHANGELOG_LINE=$(head -n 1 debian/changelog)
   CHANGELOG_DIST=$(echo "$TOP_CHANGELOG_LINE" | cut -d' ' -f3)
 
   cd - >/dev/null 2>&1
+
   # echo "Checking $CODENAME and $CHANGELOG_DIST"
   if [[ "$CHANGELOG_DIST" == *"$CODENAME"* ]]; then
     return 0
@@ -36,14 +37,42 @@ stage_source() {
   pushd .
 
   echo "Preparing source for $PACKAGE_NAME"
-  cd "$PKG_BUILD_DIR/$PACKAGE_NAME" || exit
+  cd "$PKG_BUILD_PATH/$PACKAGE_NAME" || exit
   debian_package_name=$(dpkg-parsechangelog --show-field Source)
   full_version=$(dpkg-parsechangelog --show-field Version)
   debian_version="${full_version%-*}"
-  cd "$PKG_BUILD_DIR" || exit
+  cd "$PKG_BUILD_PATH" || exit
 
   echo "Generating source tarball from git repo."
-  tar --force-local -c -z -v -f  "${debian_package_name}_${debian_version}.orig.tar.gz" --exclude .git\* --exclude /debian "$PACKAGE_NAME"
+  tar --force-local -c -z -v -f  "${debian_package_name}_${debian_version}.orig.tar.gz" --exclude .git\* --exclude debian "$PACKAGE_NAME"
+
+  if [ "$LOCAL_BUILD" == "false" ]; then
+    debian_package_name_indicator="${debian_package_name:0:1}"
+    if [ "${debian_package_name:0:3}" == "lib" ]; then
+      debian_package_name_indicator="${debian_package_name:0:4}"
+    fi
+
+    # try to download the .orig.tar.gz from existing archive, and check if they are identical or not
+    wget -O "${debian_package_name}_${debian_version}-existing.orig.tar.gz" "http://archive.regolith-desktop.com/$DISTRO/$SUITE/pool/main/${debian_package_name_indicator}/${debian_package_name}/${debian_package_name}_${debian_version}.orig.tar.gz" || true
+
+    if [ -s "${debian_package_name}_${debian_version}-existing.orig.tar.gz" ]; then
+      if ! diff <(tar -tvzf "${debian_package_name}_${debian_version}.orig.tar.gz" | awk '{printf "%10s %s\n",$3,$6}' | sort -k 2 | sed 's|\./||') <(tar -tvzf "${debian_package_name}_${debian_version}-existing.orig.tar.gz" | awk '{printf "%10s %s\n",$3,$6}' | sort -k 2 | sed 's|\./||') ; then
+        # existing .orig.tar.gz file is different that the one we just built
+        # keep the one we just built and override push it to the repository.
+        rm -f "${debian_package_name}_${debian_version}-existing.orig.tar.gz" || true
+
+        echo "SRCLOG:$DISTRO=$CODENAME=$SUITE=${debian_package_name:0:1}=${debian_package_name}=${debian_package_name}_${debian_version}=${debian_package_name}_${debian_version}.orig.tar.gz"
+      else
+        # both .orig.tar.gz files are identical!
+        # remove the one we just built and reuse the existign one.
+        rm -f "${debian_package_name}_${debian_version}.orig.tar.gz" || true
+        mv "${debian_package_name}_${debian_version}-existing.orig.tar.gz" "${debian_package_name}_${debian_version}.orig.tar.gz"
+      fi
+    else
+      # there's no existing .orig.tar.gz file! Clean up the empty downloaded file.
+      rm -f "${debian_package_name}_${debian_version}-existing.orig.tar.gz" || true
+    fi
+  fi
 
   popd
 }
@@ -53,10 +82,11 @@ build_src_package() {
 
   pushd .
   echo "Building source package $PACKAGE_NAME"
-  cd "$PKG_BUILD_DIR/$PACKAGE_NAME" || exit
+  cd "$PKG_BUILD_PATH/$PACKAGE_NAME" || exit
 
   sanitize_git
-  sudo apt update  
+
+  sudo apt update
   sudo apt build-dep -y .
   debuild -S -sa
 
@@ -68,108 +98,123 @@ build_bin_package() {
 
   pushd .
   echo "Building binary package $PACKAGE_NAME"
-  cd "$PKG_BUILD_DIR/$PACKAGE_NAME" || exit
+  cd "$PKG_BUILD_PATH/$PACKAGE_NAME" || exit
 
   debuild -sa -b
   popd
 }
 
-# Internal (private) function
-source_pkg_exists() {
-    SRC_PKG_VERSION=$(reprepro --basedir "$PKG_REPO_PATH" list "$CODENAME" "$1" | cut -d' ' -f3)
-
-    SRC_PKG_BUILD_VERSION=$(echo $2 | cut -d'-' -f1)
-    SRC_PKG_REPO_VERSION=$(echo $SRC_PKG_VERSION | cut -d'-' -f1)
-
-    if [ "$SRC_PKG_REPO_VERSION" == "$SRC_PKG_BUILD_VERSION" ]; then
-        return 0
-    else
-        return 1
-    fi
-}
-
 publish() {
-  cd "${PKG_BUILD_DIR:?}/$PACKAGE_NAME"
+  cd "${PKG_BUILD_PATH:?}/$PACKAGE_NAME"
   version=$(dpkg-parsechangelog --show-field Version)
   debian_package_name=$(dpkg-parsechangelog --show-field Source)
-  cd "$PKG_BUILD_DIR"
+  cd "$PKG_BUILD_PATH"
 
-  DEB_SRC_PKG_PATH="$PKG_BUILD_DIR/${debian_package_name}_${version}_source.changes"
+  DEB_SRC_PKG_PATH="$(pwd)/${debian_package_name}_${version}_source.changes"
 
   if [ ! -f "$DEB_SRC_PKG_PATH" ]; then
     echo "Failed to find changes file."
-  fi
-
-  if source_pkg_exists "$debian_package_name" "$version"; then
-      echo "Ignoring source package, already exists in target repository"
-      allow_failing_bin_package="true"
   else
-      echo "Ingesting source package $debian_package_name into $PKG_REPO_PATH"
-      reprepro --basedir "$PKG_REPO_PATH" include "$CODENAME" "$DEB_SRC_PKG_PATH"
-      allow_failing_bin_package="false"
+    short_version="${version%-*}"
+
+    mkdir -p $PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE
+
+    cp "$(pwd)/${debian_package_name}_${version}.dsc" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"
+    cp "$(pwd)/${debian_package_name}_${short_version}.orig.tar.gz" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"
+
+    if [ -f "$(pwd)/${debian_package_name}_${version}.debian.tar.xz" ]; then
+      cp "$(pwd)/${debian_package_name}_${version}.debian.tar.xz" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"
+    fi
+    if [ -f "$(pwd)/${debian_package_name}_${version}.diff.gz" ]; then
+      cp "$(pwd)/${debian_package_name}_${version}.diff.gz" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"
+    fi
+
+    if [ "$LOCAL_BUILD" == "false" ] && [ "$SUITE" == "stable" ]; then
+      mkdir -p "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$COMPONENT"
+      cd "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$COMPONENT" >/dev/null 2>&1
+
+      ln "../$SUITE/${debian_package_name}_${version}.dsc" .
+      ln "../$SUITE/${debian_package_name}_${short_version}.orig.tar.gz" .
+
+      if [ -f "../$SUITE/${debian_package_name}_${version}.debian.tar.xz" ]; then
+        ln "../$SUITE/${debian_package_name}_${version}.debian.tar.xz" .
+      fi
+      if [ -f "../$SUITE/${debian_package_name}_${version}.diff.gz" ]; then
+        ln "../$SUITE/${debian_package_name}_${version}.diff.gz" .
+      fi
+
+      cd - >/dev/null 2>&1
+    fi
+
+    echo "Publishing source package $debian_package_name into $PKG_PUBLISH_PATH"
   fi
 
-  DEB_CONTROL_FILE="$PKG_BUILD_DIR/$PACKAGE_NAME/debian/control"
+  DEB_CONTROL_FILE="$PKG_BUILD_PATH/$PACKAGE_NAME/debian/control"
   ALL_ARCH="$ARCH,all"
 
   for target_arch in $(echo $ALL_ARCH | sed "s/,/ /g"); do
-      cat "$DEB_CONTROL_FILE" | grep ^Package: | cut -d' ' -f2 | while read -r bin_pkg; do
-          DEB_BIN_PKG_PATH="$(pwd)/${bin_pkg}_${version}_${target_arch}.deb"
-          if [ -f "$DEB_BIN_PKG_PATH" ]; then
-              if [ "$allow_failing_bin_package" == "true" ]; then                
-                # If the source package/version already exists, allow the bin package build to fail (already exists)
-                reprepro --basedir "$PKG_REPO_PATH" includedeb "$CODENAME" "$DEB_BIN_PKG_PATH" || true
-                echo "Ingested binary package $DEB_BIN_PKG_PATH into $PKG_REPO_PATH"
-              else
-                reprepro --basedir "$PKG_REPO_PATH" includedeb "$CODENAME" "$DEB_BIN_PKG_PATH"
-                echo "Ingested binary package $DEB_BIN_PKG_PATH into $PKG_REPO_PATH"
-              fi
-              echo "CHLOG:Published ${bin_pkg}_${version}_${target_arch}.deb in $STAGE $DISTRO $CODENAME $ARCH from $PKG_LINE"
-          else
-              echo "Package $bin_pkg does not exist for $target_arch"
-          fi
-      done
+    cat "$DEB_CONTROL_FILE" | grep ^Package: | cut -d' ' -f2 | while read -r bin_pkg; do
+      DEB_BIN_PKG_PATH="$(pwd)/${bin_pkg}_${version}_${target_arch}.deb"
+
+      if [ -f "$DEB_BIN_PKG_PATH" ]; then
+        mkdir -p $PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE
+        cp "$DEB_BIN_PKG_PATH" "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$SUITE"
+
+        if [ "$LOCAL_BUILD" == "false" ] && [ "$SUITE" == "stable" ]; then
+          mkdir -p "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$COMPONENT"
+          cd "$PKG_PUBLISH_PATH/$DISTRO/$CODENAME/$COMPONENT" >/dev/null 2>&1
+          ln "../$SUITE/${bin_pkg}_${version}_${target_arch}.deb" .
+          cd - >/dev/null 2>&1
+        fi
+
+        echo "Publishing deb package(s) $bin_pkg into $PKG_PUBLISH_PATH"
+        echo "CHLOG:Published ${bin_pkg}_${version}_${target_arch}.deb in $DISTRO/$CODENAME/$STAGE from $PKG_LINE"
+      else
+        echo "Package $bin_pkg does not exist for $target_arch"
+      fi
+    done
   done
+}
+
+archive_setup_scripts() {
+  # Following allows for internal dependencies
+  rm /tmp/Release || true
+  wget -P /tmp "http://archive.regolith-desktop.com/$DISTRO/$SUITE/dists/$CODENAME/Release" || true
+
+  if [ -s /tmp/Release ]; then
+    rm /tmp/Release
 
-  # Preliminary launchpad.net integration. Ideally gated from action rather than distro/stage
-  # Only packages for Ubuntu distro should go to launchpad
-  # Arch is set to amd64 because launchpad handles cross arch builds internally
-  # Remove STAGE check once testing and release PPAs are ready
-  # if [[ "$DISTRO" == "ubuntu" && "$ARCH" == "amd64" && "$STAGE" == "unstable" ]]; then    
-  #   LAUNCHPAD_REPO="ppa:regolith-desktop/$CODENAME-$STAGE"
-  # 
-  #   dput -f $LAUNCHPAD_REPO $DEB_SRC_PKG_PATH || true
-  # 
-  #   echo "CHLOG: Published $PACKAGE_NAME to $LAUNCHPAD_REPO"
-  # fi
+    local repo_line=""
+    if [ "$LOCAL_BUILD" == "false" ] && [ "$SUITE" == "stable" ]; then
+      # fixed version component
+      repo_line="http://archive.regolith-desktop.com/$DISTRO/$SUITE $CODENAME v$COMPONENT"
+    else
+      # main component
+      repo_line="http://archive.regolith-desktop.com/$DISTRO/$SUITE $CODENAME $COMPONENT"
+    fi
+
+    echo "Adding repo to apt: $repo_line"
+    wget -qO - http://archive.regolith-desktop.com/regolith.key | sudo apt-key add -
+    echo "deb [arch=$ARCH] $repo_line" | sudo tee /etc/apt/sources.list.d/regolith.list
+
+    sudo apt update
+  fi
+
+  if [ -f "/etc/apt/sources.list.d/regolith-local.list" ]; then
+    sudo rm /etc/apt/sources.list.d/regolith-local.list
+    echo "Cleaned up temp apt repo"
+  fi
 }
 
-# Create repo dist file
-generate_reprepro_dist() {
-    echo "Origin: $PACKAGE_REPO_URL" > "$PKG_REPO_PATH/conf/distributions"
-    echo "Label: $PACKAGE_REPO_URL" >> "$PKG_REPO_PATH/conf/distributions"
-    echo "Codename: $CODENAME" >> "$PKG_REPO_PATH/conf/distributions"
-    echo "Architectures: $ARCH source" >> "$PKG_REPO_PATH/conf/distributions"
-    echo "Components: main" >> "$PKG_REPO_PATH/conf/distributions"
-    echo "Description: $STAGE $DISTRO $CODENAME $ARCH" >> "$PKG_REPO_PATH/conf/distributions"
-    echo "SignWith: $APT_KEY" >> "$PKG_REPO_PATH/conf/distributions"
+archive_cleanup_scripts() {
+  # Remove regolith repo from build system apt config
+  if [ -f /etc/apt/sources.list.d/regolith.list ]; then
+    sudo rm -f /etc/apt/sources.list.d/regolith.list
+  fi
 }
 
 # Setup debian repo
 setup() {
-  if [ ! -d "$PKG_REPO_PATH/conf" ]; then
-    echo "Creating conf dir"
-    mkdir -p "$PKG_REPO_PATH/conf"    
-  fi
-
-  if [ ! -f "$PKG_REPO_PATH/conf/distributions" ]; then
-    echo "Package metadata not found, creating conf dir"
-    generate_reprepro_dist
-    cat "$PKG_REPO_PATH/conf/distributions"
-  else
-    echo "Existing metadata:"
-    cat "$PKG_REPO_PATH/conf/distributions"
-  fi
-
   source_setup_scripts
+  archive_setup_scripts
 }
diff --git a/.github/scripts/ext-git.sh b/.github/scripts/ext-git.sh
@@ -4,24 +4,25 @@ checkout() {
   set -e
 
   if [ -z "$PACKAGE_URL" ]; then
-    echo "Package model is invalid.  Model field 'source' undefined, aborting."
+    echo "Error: package model is invalid. Model field 'source' undefined, aborting."
     exit 1
   fi
 
-  if [ -d "$PKG_BUILD_DIR/$PACKAGE_NAME" ]; then
+  if [ -d "$PKG_BUILD_PATH/$PACKAGE_NAME" ]; then
     echo "Deleting existing repo, $PACKAGE_NAME"
-    rm -Rfv "${PKG_BUILD_DIR:?}/$PACKAGE_NAME"
+    rm -Rfv "${PKG_BUILD_PATH:?}/$PACKAGE_NAME"
   fi
 
-  if [ ! -d "$PKG_BUILD_DIR" ]; then
-    echo "Creating build directory $PKG_BUILD_DIR"
-    mkdir -p "$PKG_BUILD_DIR" || {
-      echo "Failed to create build dir $PKG_BUILD_DIR, aborting."
+  if [ ! -d "$PKG_BUILD_PATH" ]; then
+    echo "Creating build directory $PKG_BUILD_PATH"
+
+    mkdir -p "$PKG_BUILD_PATH" || {
+      echo "Error: failed to create build dir $PKG_BUILD_PATH, aborting."
       exit 1
     }
   fi
 
-  cd "$PKG_BUILD_DIR" || exit
+  cd "$PKG_BUILD_PATH" || exit
   git clone --recursive "$PACKAGE_URL" -b "$PACKAGE_REF" "$PACKAGE_NAME"
 
   cd - >/dev/null 2>&1 || exit
@@ -36,4 +37,4 @@ sanitize_git() {
     rm -Rf .git
     echo "Removed $(pwd).git directory before building to appease debuild."
   fi
-}
+}
diff --git a/.github/scripts/ext-rpm.sh b/.github/scripts/ext-rpm.sh
@@ -41,6 +41,16 @@ publish() {
   : 
 }
 
+archive_setup_scripts() {
+  # Setting up local archive repo for internal dependencies
+  :
+}
+
+archive_cleanup_scripts() {
+  # Cleaning up local archive repo definition
+  :
+}
+
 # Setup debian repo
 setup() {
   :

diff --git a/.github/scripts/ext-template.sh b/.github/scripts/ext-template.sh
@@ -37,14 +37,18 @@ publish() {
   echo todo
 }
 
-# Create repo dist file - for new repository
-generate_reprepro_dist() {
-    echo todo
+archive_setup_scripts() {
+  echo todo
+}
+
+archive_cleanup_scripts() {
+  echo todo
 }
 
 # Setup repo layout
 setup() {
   echo todo
 
   source_setup_scripts
+  archive_setup_scripts
 }