refactor debian packager

motivation: add docker support + prepare for sharing

changes:
* add docker and docker compose setup to build in docker
* move shared scripts and metadata into the "shared" directory
* rename getsource.sh to build_source_package.sh and udpate it to handle the new directory structure
* create build_deb.sh driver script to do the package building end-to-end
* update readme with details

Author: tomerd

platforms/Linux/DEB/Shared/build_deb.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -eux
+
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# load version definitions
+. ${here}/versions.sh
+
+# working in /tmp since docker file sharing makes this very slow to do dirctly on the share
+staging_dir=/tmp/swift-deb-builder
+package_dir=${staging_dir}/swiftlang-${debversion}
+
+# clean
+#rm -rf ${staging_dir} && mkdir -p ${staging_dir}
+rm -rf ${package_dir} && mkdir -p ${package_dir}
+
+# copy control files to pakcage build directory
+cp -r ${here}/debian ${package_dir}/
+cp -r ${package_dir}/debian/control.in ${package_dir}/debian/control
+
+# build the source package
+${here}/build_source_package.sh ${staging_dir}
+
+# install the build dependencies
+cd ${staging_dir}
+mk-build-deps --install ${package_dir}/debian/control.in --tool 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends'
+
+# build the installable package
+# TODO: add signing key information
+cd ${package_dir}
+DEB_BUILD_OPTIONS=parallel=64 debuild
+
+# copy the final packages to /output
+cd ${staging_dir}
+cp *.deb /output/
+cp *.ddeb /output/
+cp *.dsc /output/
+cp *.tar.* /output/

platforms/Linux/DEB/Ubuntu/focal/debian/getsource.sh renamed to platforms/Linux/DEB/Shared/build_source_package.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # This helper script will fetch the Swift sources, save the
 # tarballs following the appropriate naming convention, and
@@ -10,7 +10,7 @@
 
 # Create swiftlang-X.Y.Z, unpack the debian tarball within,
 # proceed to adding the appropriate entry to the Debian
-# changelog, edit debian/source-version.sh, then run this
+# changelog, edit /source-version.sh, then run this
 # script from the directory containing swiftlang-X.Y.Z.
 #   $ mkdir swiftlang-X.Y.Z
 #   $ cd swiftlang-X.Y.Z
@@ -27,17 +27,22 @@
 #   $ cd swiftlang-X.Y.Z
 #   $ DEB_BUILD_OPTIONS=parallel=64 debuild
 
-set -eu
+set -eux
 
-. $(dirname $0)/source-versions.sh
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# load version definitions
+. ${here}/versions.sh
+
+staging_dir=$1
+package_dir=$staging_dir/swiftlang-${debversion}
 
 get_component ()
 {
     component=$1
     url="$2"
 
-    dest=swiftlang_${debversion}.orig-${component}
-
+    dest=${staging_dir}/swiftlang_${debversion}.orig-${component}
     echo "Downloading ${component} from ${url}"
 
     case "${url}" in
@@ -51,7 +56,10 @@ get_component ()
 
 	*.tar.gz)
 	    dest=${dest}.tar.gz
-	    curl -L -o ${dest} "${url}"
+      # temp
+      if [ ! -f ${dest} ]; then
+	      curl -L -o ${dest} "${url}"
+      fi
 	    ;;
 
 	*.tar.xz)
@@ -69,10 +77,12 @@ get_component ()
 	    exit 1
     esac
 
-    mkdir swiftlang-${debversion}/${component}
-    tar -C swiftlang-${debversion}/${component} --strip-components=1 -axf ${dest}
+    echo "Extracting ${component}"
+    mkdir ${package_dir}/${component}
+    tar -C ${package_dir}/${component} --strip-components=1 -axf ${dest}
 }
 
+# download
 get_component swift https://github.com/apple/swift/archive/swift-${swift_version}.tar.gz
 get_component swift-corelibs-libdispatch https://github.com/apple/swift-corelibs-libdispatch/archive/swift-${swift_version}.tar.gz
 get_component swift-corelibs-foundation https://github.com/apple/swift-corelibs-foundation/archive/swift-${swift_version}.tar.gz
@@ -109,10 +119,10 @@ get_component swift-lmdb https://github.com/apple/swift-lmdb/archive/swift-${swi
 get_component swift-markdown https://github.com/apple/swift-markdown/archive/swift-${swift_version}.tar.gz
 
 # Refresh patches, if any
-if [ -s swiftlang-${debversion}/debian/patches/series ]; then
-    cd swiftlang-${debversion}
+if [ -s ${package_dir}/debian/patches/series ]; then
+    cd ${package_dir}
 
-    export QUILT_PATCHES=debian/patches 
+    export QUILT_PATCHES=debian/patches
     export QUILT_REFRESH_ARGS="-p ab --no-timestamps --no-index"
 
     while quilt push; do quilt refresh; done
@@ -121,4 +131,6 @@ if [ -s swiftlang-${debversion}/debian/patches/series ]; then
     cd -
 fi
 
-dpkg-source --create-empty-orig -b swiftlang-${debversion}
+# create a source package
+cd $staging_dir
+dpkg-source --create-empty-orig -b ${package_dir}

platforms/Linux/DEB/Ubuntu/focal/debian/copyright renamed to platforms/Linux/DEB/Shared/copyright

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 debversion=5.6.0
 

platforms/Linux/DEB/Ubuntu/focal/debian/source-versions.sh renamed to platforms/Linux/DEB/Shared/versions.sh

@@ -1 +1,71 @@
 ## Ubuntu
+
+Ubuntu uses the [Deb package format]() to install software packages.
+The Swift Deb package can be built either by creating a Linux container image or manually on a computer running Ubuntu.
+
+There are separate directories for each version of Ubuntu.
+The instructions below are applicable to all versions.
+
+## Importand file and directories
+
+**build_deb.sh**
+Driver file to build the deb package
+
+**build_source.sh**
+Driver file to build the deb source package (which is a step in `build_deb`)
+
+**patches/*.patch**
+Any post-release patches that have not yet been merged upstream that are temporarily necessary to build Swift.
+
+**control.in**
+Debian package metadata, including the `BuildDepends` and `Depends` definitions.
+
+**changelog**
+Debian package changelog
+
+**rules**
+Debian package recipe 
+
+**Dockerfile**
+Defines the base docker image to run the install scripts in.
+
+**docker-compose**
+Defines docker compose tasks to drive the pacakge build in Docker.
+
+## Importand file and directories
+
+**Shared/version.sh**
+Shell fragment versions.sh containing version information for all source components, and the Debian package upstream version (debversion).
+
+**Shared/copyright**
+Copyright information
+
+### Building with docker-compose
+
+* to run the build end-to-end
+
+```
+docker-compose run build
+```
+
+* to enter the docker env in shell mode
+
+```
+docker-compose run shell
+```
+
+then you can run `./build_deb.sh` to run the build manually inside the docker
+
+
+* to rebuild the base image
+
+```
+docker-compose build --pull
+```
+
+note this still uses the docker cache, so will rebuild only if the version of the underlying base image changed upstream
+
+### Building locally on an Ubuntu machine
+
+1. Install required development tools (see Dockerfile)
+2. Run `./build_deb.sh`

platforms/Linux/DEB/Ubuntu/README.md

@@ -0,0 +1,21 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+FROM ubuntu:focal
+LABEL PURPOSE="This image is configured to build Swift for the version of Ubuntu listed above"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+
+# Required deb packaging tools
+RUN apt-get install -y curl devscripts equivs quilt tar
+
+# Optimization: Install Swift build requirements listed in the control file
+ADD control.in /tmp/control.in
+RUN mk-build-deps --install /tmp/control.in --tool 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends'

platforms/Linux/DEB/Ubuntu/focal/Dockerfile

@@ -0,0 +1 @@
+../../Shared/build_deb.sh

platforms/Linux/DEB/Ubuntu/focal/README.md

@@ -0,0 +1 @@
+../../Shared/build_source_package.sh

platforms/Linux/DEB/Ubuntu/focal/build_deb.sh

@@ -0,0 +1,47 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+# this setup is designed to build the RPM with docker
+# usage:
+# docker-compose -f platforms/Linux/DEB/ubuntu/focal/docker-compose.yaml build
+# to shell into the container for debugging purposes:
+# docker-compose -f platforms/Linux/DEB/ubuntu/focal/docker-compose.yaml run build
+
+version: "2"
+
+services:
+
+  docker-setup:
+    image: ubuntu-focal-deb-builder
+    build:
+      context: .
+      dockerfile: Dockerfile
+
+  common: &common
+    image: ubuntu-focal-deb-builder
+    depends_on: [docker-setup]
+    volumes:
+      - ../../Ubuntu/focal:/code/Ubuntu/focal:z
+      - ../../Shared:/code/Shared:z
+      - ./.output:/output:z
+    working_dir: /code/Ubuntu/focal
+    cap_drop:
+      - CAP_NET_RAW
+      - CAP_NET_BIND_SERVICE
+
+  build:
+    <<: *common
+    command: /bin/bash -cl "./build_deb.sh"
+
+  #createrepo:
+  #  <<: *common
+  #  command: /bin/bash -cl "./createrepo_rpm.sh"
+
+  shell:
+    <<: *common
+    entrypoint: /bin/bash -l

platforms/Linux/DEB/Ubuntu/focal/build_source_package.sh

@@ -0,0 +1 @@
+../../Shared/versions.sh