[wip] add docker support to debian packager

platforms/Linux/DEB/Shared/build_deb.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -eux
+
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# load version definitions
+. ${here}/versions.sh
+
+# working in /tmp since docker file sharing makes this very slow to do dirctly on the share
+staging_dir=/tmp/swift-deb-builder
+package_dir=${staging_dir}/swiftlang-${debversion}
+
+# clean
+#rm -rf ${staging_dir} && mkdir -p ${staging_dir}
+rm -rf ${package_dir} && mkdir -p ${package_dir}
+
+# copy control files to pakcage build directory
+cp -r ${here}/debian ${package_dir}/
+cp -r ${package_dir}/debian/control.in ${package_dir}/debian/control
+
+# build the source package
+${here}/build_source_package.sh ${staging_dir}
+
+# install the build dependencies
+cd ${staging_dir}
+mk-build-deps --install ${package_dir}/debian/control.in --tool 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends'
+
+# build the installable package
+# TODO: add signing key information
+cd ${package_dir}
+DEB_BUILD_OPTIONS=parallel=64 debuild
+
+# copy the final packages to /output
+cd ${staging_dir}
+cp *.deb /output/
+cp *.ddeb /output/
+cp *.dsc /output/
+cp *.tar.* /output/

platforms/Linux/DEB/Shared/build_source_package.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# This helper script will fetch the Swift sources, save the
+# tarballs following the appropriate naming convention, and
+# build a Debian source package.
+
+# You will need the devscripts and quilt packages installed
+# to run this script and build the packages following these
+# instructions.
+
+# Create swiftlang-X.Y.Z, unpack the debian tarball within,
+# proceed to adding the appropriate entry to the Debian
+# changelog, edit /source-version.sh, then run this
+# script from the directory containing swiftlang-X.Y.Z.
+#   $ mkdir swiftlang-X.Y.Z
+#   $ cd swiftlang-X.Y.Z
+#   $ tar xvaf ../swiftlang_A.B.C-D.debian.tar.xz
+#   $ dch -v X.Y.Z-1
+#   $ cd ..
+#   $ swiftlang-X.Y.Z/debian/getsource.sh
+#
+# This will download all the necessary sources, unpack them
+# as required, refresh any Debian patches, then create the
+# Debian source package.
+
+# If all is well, proceed with building the packages.
+#   $ cd swiftlang-X.Y.Z
+#   $ DEB_BUILD_OPTIONS=parallel=64 debuild
+
+set -eux
+
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# load version definitions
+. ${here}/versions.sh
+
+staging_dir=$1
+package_dir=$staging_dir/swiftlang-${debversion}
+
+get_component ()
+{
+    component=$1
+    url="$2"
+
+    dest=${staging_dir}/swiftlang_${debversion}.orig-${component}
+    echo "Downloading ${component} from ${url}"
+
+    case "${url}" in
+	*.zip)
+	    tmpdest=${component}.zip
+	    dest=${dest}.tar.gz
+
+	    curl -L -o ${tmpdest} "${url}"
+	    mk-origtargz --package swiftlang -v ${debversion} -c ${component} --rename --compression gzip ${tmpdest}
+	    ;;
+
+	*.tar.gz)
+	    dest=${dest}.tar.gz
+      # temp
+      if [ ! -f ${dest} ]; then
+	      curl -L -o ${dest} "${url}"
+      fi
+	    ;;
+
+	*.tar.xz)
+	    dest=${dest}.tar.xz
+	    curl -L -o ${dest} "${url}"
+	    ;;
+
+	*.tar.bz2)
+	    dest=${dest}.tar.bz2
+	    curl -L -o ${dest} "${url}"
+	    ;;
+
+	*)
+	    echo "Source archive not in a suitable format for Debian sources: ${url}" >&2
+	    exit 1
+    esac
+
+    echo "Extracting ${component}"
+    mkdir ${package_dir}/${component}
+    tar -C ${package_dir}/${component} --strip-components=1 -axf ${dest}
+}
+
+# download
+get_component swift https://github.com/apple/swift/archive/swift-${swift_version}.tar.gz
+get_component swift-corelibs-libdispatch https://github.com/apple/swift-corelibs-libdispatch/archive/swift-${swift_version}.tar.gz
+get_component swift-corelibs-foundation https://github.com/apple/swift-corelibs-foundation/archive/swift-${swift_version}.tar.gz
+get_component swift-integration-tests https://github.com/apple/swift-integration-tests/archive/swift-${swift_version}.tar.gz
+get_component swift-corelibs-xctest https://github.com/apple/swift-corelibs-xctest/archive/swift-${swift_version}.tar.gz
+get_component swiftpm https://github.com/apple/swift-package-manager/archive/swift-${swift_version}.tar.gz
+get_component llbuild https://github.com/apple/swift-llbuild/archive/swift-${swift_version}.tar.gz
+get_component cmark https://github.com/apple/swift-cmark/archive/swift-${swift_version}.tar.gz
+get_component swift-xcode-playground-support https://github.com/apple/swift-xcode-playground-support/archive/swift-${swift_version}.tar.gz
+get_component sourcekit-lsp https://github.com/apple/sourcekit-lsp/archive/swift-${swift_version}.tar.gz
+get_component indexstore-db https://github.com/apple/indexstore-db/archive/swift-${swift_version}.tar.gz
+get_component llvm-project https://github.com/apple/llvm-project/archive/swift-${swift_version}.tar.gz
+get_component swift-tools-support-core https://github.com/apple/swift-tools-support-core/archive/swift-${swift_version}.tar.gz
+get_component swift-argument-parser https://github.com/apple/swift-argument-parser/archive/${swift_argument_parser_version}.tar.gz
+get_component swift-driver https://github.com/apple/swift-driver/archive/swift-${swift_version}.tar.gz
+get_component icu https://github.com/unicode-org/icu/archive/release-${icu_version}.tar.gz
+get_component swift-crypto https://github.com/apple/swift-crypto/archive/refs/tags/${swift_crypto_version}.tar.gz
+get_component ninja https://github.com/ninja-build/ninja/archive/refs/tags/v${ninja_version}.tar.gz
+get_component cmake https://github.com/KitWare/CMake/archive/refs/tags/v${cmake_version}.tar.gz
+get_component swift-syntax https://github.com/apple/swift-syntax/archive/swift-${swift_version}.tar.gz
+get_component yams https://github.com/jpsim/Yams/archive/${yams_version}.tar.gz
+get_component swift-atomics https://github.com/apple/swift-atomics/archive/${swift_atomics_version}.tar.gz
+get_component swift-cmark-gfm https://github.com/apple/swift-cmark/archive/swift-${swift_version}-gfm.tar.gz
+get_component swift-docc https://github.com/apple/swift-docc/archive/swift-${swift_version}.tar.gz
+get_component swift-docc-render-artifact https://github.com/apple/swift-docc-render-artifact/archive/swift-${swift_version}.tar.gz
+get_component swift-docc-symbolkit https://github.com/apple/swift-docc-symbolkit/archive/swift-${swift_version}.tar.gz
+get_component swift-collections https://github.com/apple/swift-collections/archive/${swift_collections_version}.tar.gz
+get_component swift-numerics https://github.com/apple/swift-numerics/archive/${swift_numerics_version}.tar.gz
+get_component swift-system https://github.com/apple/swift-system/archive/${swift_system_version}.tar.gz
+get_component swift-nio https://github.com/apple/swift-nio/archive/${swift_nio_version}.tar.gz
+get_component swift-nio-ssl https://github.com/apple/swift-nio-ssl/archive/${swift_nio_ssl_version}.tar.gz
+get_component swift-format https://github.com/apple/swift-format/archive/swift-${swift_version}.tar.gz
+get_component swift-lmdb https://github.com/apple/swift-lmdb/archive/swift-${swift_version}.tar.gz
+get_component swift-markdown https://github.com/apple/swift-markdown/archive/swift-${swift_version}.tar.gz
+
+# Refresh patches, if any
+if [ -s ${package_dir}/debian/patches/series ]; then
+    cd ${package_dir}
+
+    export QUILT_PATCHES=debian/patches
+    export QUILT_REFRESH_ARGS="-p ab --no-timestamps --no-index"
+
+    while quilt push; do quilt refresh; done
+    quilt pop -a
+
+    cd -
+fi
+
+# create a source package
+cd $staging_dir
+dpkg-source --create-empty-orig -b ${package_dir}

platforms/Linux/DEB/Shared/copyright

@@ -0,0 +1,38 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: swift
+Upstream-Contact: <preferred name and address to reach the upstream project>
+Source: <url://example.com>
+
+Files: *
+Copyright: <years> <put author's name and email here>
+           <years> <likewise for another author>
+License: Apache-2.0
+
+Files: debian/*
+Copyright: <years> Author Name <[email protected]>
+License: Apache-2.0
+
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+ https://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the complete text of the Apache version 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
+
+# Please also look if there are files or directories which have a
+# different copyright/license attached and list them here.
+# Please avoid picking licenses with terms that are more restrictive than the
+# packaged work, as it may make Debian's contributions unacceptable upstream.
+#
+# If you need, there are some extra license texts available in two places:
+#   /usr/share/debhelper/dh_make/licenses/
+#   /usr/share/common-licenses/

platforms/Linux/DEB/Shared/versions.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+debversion=5.6.0
+
+swift_version=5.6-RELEASE
+icu_version=65-1
+yams_version=4.0.2
+swift_argument_parser_version=1.0.3
+swift_crypto_version=1.1.5
+ninja_version=1.10.2
+cmake_version=3.19.6
+swift_atomics_version=1.0.2
+swift_collections_version=1.0.1
+swift_numerics_version=1.0.1
+swift_system_version=1.1.1
+swift_nio_version=2.31.2
+swift_nio_ssl_version=2.15.0

platforms/Linux/DEB/Ubuntu/README.md

@@ -1 +1,71 @@
 ## Ubuntu
+
+Ubuntu uses the [Deb package format]() to install software packages.
+The Swift Deb package can be built either by creating a Linux container image or manually on a computer running Ubuntu.
+
+There are separate directories for each version of Ubuntu.
+The instructions below are applicable to all versions.
+
+## Importand file and directories
+
+**build_deb.sh**
+Driver file to build the deb package
+
+**build_source.sh**
+Driver file to build the deb source package (which is a step in `build_deb`)
+
+**patches/*.patch**
+Any post-release patches that have not yet been merged upstream that are temporarily necessary to build Swift.
+
+**control.in**
+Debian package metadata, including the `BuildDepends` and `Depends` definitions.
+
+**changelog**
+Debian package changelog
+
+**rules**
+Debian package recipe 
+
+**Dockerfile**
+Defines the base docker image to run the install scripts in.
+
+**docker-compose**
+Defines docker compose tasks to drive the pacakge build in Docker.
+
+## Importand file and directories
+
+**Shared/version.sh**
+Shell fragment versions.sh containing version information for all source components, and the Debian package upstream version (debversion).
+
+**Shared/copyright**
+Copyright information
+
+### Building with docker-compose
+
+* to run the build end-to-end
+
+```
+docker-compose run build
+```
+
+* to enter the docker env in shell mode
+
+```
+docker-compose run shell
+```
+
+then you can run `./build_deb.sh` to run the build manually inside the docker
+
+
+* to rebuild the base image
+
+```
+docker-compose build --pull
+```
+
+note this still uses the docker cache, so will rebuild only if the version of the underlying base image changed upstream
+
+### Building locally on an Ubuntu machine
+
+1. Install required development tools (see Dockerfile)
+2. Run `./build_deb.sh`

platforms/Linux/DEB/Ubuntu/focal/Dockerfile

@@ -0,0 +1,21 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2022 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+FROM ubuntu:focal
+LABEL PURPOSE="This image is configured to build Swift for the version of Ubuntu listed above"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+
+# Required deb packaging tools
+RUN apt-get install -y curl devscripts equivs quilt tar
+
+# Optimization: Install Swift build requirements listed in the control file
+ADD control.in /tmp/control.in
+RUN mk-build-deps --install /tmp/control.in --tool 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends'

platforms/Linux/DEB/Ubuntu/focal/build_deb.sh

@@ -0,0 +1 @@
+../../Shared/build_deb.sh

platforms/Linux/DEB/Ubuntu/focal/build_source_package.sh

@@ -0,0 +1 @@
+../../Shared/build_source_package.sh

platforms/Linux/DEB/Ubuntu/focal/debian/changelog

@@ -0,0 +1,17 @@
+swiftlang (5.6.0-1) focal; urgency=medium
+
+  * New upstream release.
+
+ -- Julien Blache <[email protected]>  Tue, 15 Mar 2022 16:36:34 -0700
+
+swiftlang (5.5.3-2) focal; urgency=medium
+
+  * Add a default swiftlang package with symlinks in the PATH.
+
+ -- Julien Blache <[email protected]>  Fri, 11 Mar 2022 15:04:24 -0800
+
+swiftlang (5.5.3-1) focal; urgency=medium
+
+  * Initial release.
+
+ -- Julien Blache <[email protected]>  Tue, 01 Mar 2022 15:04:23 -0800