Run the compiler validation suite for Android

.github/workflows/pull_request.yml

@@ -23,27 +23,29 @@ jobs:
             *.log
 
    android-build:
-     name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK
+     name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK (build-compiler=${{ matrix.build-compiler }})
      strategy:
        fail-fast: false
        matrix:
          #build-type: ['docker']
-         build-type: ['docker', 'local']
+         build-type: ['docker']
+         #build-compiler: ['0', '1']
+         build-compiler: ['1']
          # blank arch builds all (aarch64,x86_64,armv7)
-         #arch: ['']
+         arch: ['']
          # builds only x86_64 to speed up the validation
          #arch: ['x86_64']
          # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs
-         arch: ['x86_64', '']
-         swift-version: ['release', 'swift-6.2-branch', 'development']
+         #arch: ['x86_64', '']
+         swift-version: ['swift-6.2-branch', 'development']
      runs-on: ubuntu-24.04
      steps:
       - name: Free Disk Space
         run: |
           df -h
           # brings available space from 25G to 32G
           # otherwise we sometimes run out of space during the build
-          sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache /usr/local/graalvm/ /usr/local/.ghcup/ /usr/local/lib/node_modules /usr/local/share/boost
           sudo docker image prune --all --force
           sudo docker builder prune -a
           df -h
@@ -53,6 +55,8 @@ jobs:
           # these variabes are used by build-docker and build-local
           # to determine which Swift version to build for
           echo "BUILD_SCHEME=${{ matrix.swift-version }}" >> $GITHUB_ENV
+          # pass the build-compiler matrix through to the build script
+          echo "BUILD_COMPILER=${{ matrix.build-compiler }}" >> $GITHUB_ENV
           echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV
           echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV
       - name: Checkout repository
@@ -61,7 +65,7 @@ jobs:
         if: ${{ matrix.build-type == 'local' }}
         working-directory: swift-ci/sdks/android
         run: |
-          sudo apt install -q ninja-build patchelf
+          sudo apt install -q patchelf build-essential cmake ninja-build python3 golang git gnupg2 libcurl4-openssl-dev libedit-dev libicu-dev libncurses5-dev libpython3-dev libsqlite3-dev libxml2-dev rsync uuid-dev uuid-runtime tzdata curl unzip
           ./build-local ${BUILD_SCHEME} ${WORKDIR}
       - name: Build Android SDK (Docker)
         if: ${{ matrix.build-type == 'docker' }}
@@ -99,6 +103,9 @@ jobs:
           if [[ "${{ matrix.build-type }}" == 'local' ]]; then
             ARTIFACT_NAME="${ARTIFACT_NAME}-local"
           fi
+          if [[ "${{ matrix.build-compiler }}" == '1' ]]; then
+            ARTIFACT_NAME="${ARTIFACT_NAME}-hostbuild"
+          fi
           # artifacts need a unique name so we suffix with the matrix arch(s)
           if [[ ! -z "${{ matrix.arch }}" ]]; then
             ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')"

README.md

@@ -2,9 +2,9 @@
 
 <img src="https://swift.org/assets/images/swift.svg" alt="Swift logo" height="70" >
 
-### Docker images for [Swift](https://swift.org).
+###  images for [Swift](https://swift.org).
 
-#### You can find the Docker Hub repo here: [https://hub.docker.com/_/swift/](https://hub.docker.com/_/swift/)
+#### You can find the  Hub repo here: [https://hub.docker.com/_/swift/](https://hub.docker.com/_/swift/)
 
 #### Nightly image tags are published here: [https://hub.docker.com/r/swiftlang/swift](https://hub.docker.com/r/swiftlang/swift)
 

swift-ci/sdks/android/Dockerfile

@@ -18,6 +18,7 @@ ARG SWIFT_TOOLCHAIN_URL=
 RUN apt-get -q update \
     && DEBIAN_FRONTEND=noninteractive apt-get -q install -y \
     build-essential \
+    clang \
     cmake \
     ninja-build \
     python3 \
@@ -54,7 +55,8 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
 COPY scripts/install-swift.sh /scripts/install-swift.sh
 RUN chmod ugo+x /scripts/install-swift.sh
 RUN /scripts/install-swift.sh /usr/local/swift
-ENV PATH="/usr/local/swift/bin:${PATH}"
+#ENV PATH="/usr/local/swift/bin:${PATH}"
+ENV PATH="${PATH}:/usr/local/swift/bin"
 
 ARG ANDROID_NDK_VERSION=
 

swift-ci/sdks/android/build-docker

@@ -59,9 +59,10 @@ $DOCKER run -i --rm \
           -v ${WORKDIR}/products:/products:rw \
           ${CONTAINER_NAME} \
           /scripts/build.sh \
-            --source-dir /source \
-            --products-dir /products \
-            --host-toolchain /usr/local/swift \
-            --android-api ${ANDROID_API} \
-            --ndk-home /usr/local/ndk/${ANDROID_NDK_VERSION} \
-            --archs ${TARGET_ARCHS}
+            --source-dir "/source" \
+            --products-dir "/products" \
+            --host-toolchain "/usr/local/swift" \
+            --build-compiler "${BUILD_COMPILER}" \
+            --android-api "${ANDROID_API}" \
+            --ndk-home "/usr/local/ndk/${ANDROID_NDK_VERSION}" \
+            --archs "${TARGET_ARCHS}"

swift-ci/sdks/android/build-local

@@ -25,16 +25,18 @@ WORKDIR=$(realpath ${WORKDIR})
 HOST_OS=ubuntu$(lsb_release -sr)
 source ./scripts/toolchain-vars.sh
 
-SWIFT_ROOT=${WORKDIR}/host-toolchain
-HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr
-if [[ ! -d "$HOST_TOOLCHAIN" ]]; then
-    ./scripts/install-swift.sh ${HOST_TOOLCHAIN}
-fi
+if [[ "${BUILD_COMPILER}" != "1" ]]; then
+    SWIFT_ROOT=${WORKDIR}/host-toolchain
+    HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr
+    if [[ ! -d "$HOST_TOOLCHAIN" ]]; then
+        ./scripts/install-swift.sh ${HOST_TOOLCHAIN}
+    fi
 
-$HOST_TOOLCHAIN/bin/swift --version
+    $HOST_TOOLCHAIN/bin/swift --version
 
-# ensure the correct Swift is first in the PATH
-export PATH=$HOST_TOOLCHAIN/bin:$PATH
+    # ensure the correct Swift is first in the PATH
+    export PATH=$HOST_TOOLCHAIN/bin:$PATH
+fi
 
 export ANDROID_NDK_HOME=${WORKDIR}/ndk/${ANDROID_NDK_VERSION}
 
@@ -55,10 +57,11 @@ ${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift
 mkdir -p ${WORKDIR}/products
 
 ./scripts/build.sh \
-    --source-dir ${WORKDIR}/source \
-    --products-dir ${WORKDIR}/products \
-    --build-dir ${WORKDIR}/build \
-    --host-toolchain ${HOST_TOOLCHAIN} \
-    --android-api ${ANDROID_API} \
-    --ndk-home ${ANDROID_NDK_HOME} \
-    --archs ${TARGET_ARCHS}
+    --source-dir "${WORKDIR}/source" \
+    --products-dir "${WORKDIR}/products" \
+    --build-dir "${WORKDIR}/build" \
+    --build-compiler "${BUILD_COMPILER}" \
+    --host-toolchain "${HOST_TOOLCHAIN}" \
+    --android-api "${ANDROID_API}" \
+    --ndk-home "${ANDROID_NDK_HOME}" \
+    --archs "${TARGET_ARCHS}"

swift-ci/sdks/android/scripts/build.sh

@@ -50,7 +50,7 @@ function groupend {
 
 function usage {
     cat <<EOF
-usage: build.sh --source-dir <path> --products-dir <path> --ndk-home <path> --host-toolchain <path>
+usage: build.sh --source-dir <path> --products-dir <path> --ndk-home <path>
                 [--name <sdk-name>] [--version <version>] [--build-dir <path>]
                 [--archs <arch>[,<arch> ...]]
 
@@ -63,6 +63,7 @@ Options:
   --source-dir <path>   Specify the path in which the sources can be found.
   --ndk-home <path>     Specify the path to the Android NDK
   --host-toolchain <tc> Specify the path to the host Swift toolchain
+  --build-compiler <bc> Whether to build and validate the host compiler
   --products-dir <path> Specify the path in which the products should be written.
   --build-dir <path>    Specify the path in which intermediates should be stored.
   --android-api <api>   Specify the Android API level
@@ -130,6 +131,8 @@ while [ "$#" -gt 0 ]; do
             ndk_home="$2"; shift ;;
         --host-toolchain)
             host_toolchain="$2"; shift ;;
+        --build-compiler)
+            build_compiler="$2"; shift ;;
         --build-dir)
             build_dir="$2"; shift ;;
         --android-api)
@@ -155,7 +158,7 @@ done
 # Change the commas for spaces
 archs="${archs//,/ }"
 
-if [[ -z "$source_dir" || -z "$products_dir" || -z "$ndk_home" || -z "$host_toolchain" ]]; then
+if [[ -z "$source_dir" || -z "$products_dir" || -z "$ndk_home" ]]; then
     usage
     exit 1
 fi
@@ -234,8 +237,10 @@ export ANDROID_NDK_HOME=$ndk_home
 export ANDROID_NDK=$ndk_home
 
 echo "Swift found at ${swift_dir}"
-echo "Host toolchain found at ${host_toolchain}"
-${host_toolchain}/bin/swift --version
+if [[ ! -z "${host_toolchain}" ]]; then
+    echo "Host toolchain found at ${host_toolchain}"
+    ${host_toolchain}/bin/swift --version
+fi
 echo "Android NDK found at ${ndk_home}"
 ${ndk_installation}/bin/clang --version
 echo "Building for ${archs}"
@@ -262,11 +267,31 @@ function run() {
 
 for arch in $archs; do
     case $arch in
-        armv7) target_host="arm-linux-androideabi"; compiler_target_host="armv7a-linux-androideabi$android_api"; android_abi="armeabi-v7a" ;;
-        aarch64) target_host="aarch64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="arm64-v8a" ;;
-        x86_64) target_host="x86_64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="x86_64" ;;
-        x86) target_host="x86-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="x86" ;;
-        *) echo "Unknown architecture '$1'"; usage; exit 0 ;;
+        armv7)
+            target_host="arm-linux-androideabi"
+            compiler_target_host="armv7a-linux-androideabi$android_api"
+            android_abi="armeabi-v7a"
+            ;;
+        aarch64)
+            target_host="aarch64-linux-android"
+            compiler_target_host="$target_host$android_api"
+            android_abi="arm64-v8a"
+            ;;
+        x86_64)
+            target_host="x86_64-linux-android"
+            compiler_target_host="$target_host$android_api"
+            android_abi="x86_64"
+            ;;
+        x86)
+            target_host="x86-linux-android"
+            compiler_target_host="$target_host$android_api"
+            android_abi="x86"
+            ;;
+        *)
+            echo "Unknown architecture '$1'"
+            usage
+            exit 0
+            ;;
     esac
 
     sdk_root=${build_dir}/sdk_root/${arch}
@@ -378,35 +403,61 @@ for arch in $archs; do
             RelWithDebInfo) build_type_flag="--release-debuginfo" ;;
         esac
 
+        case $build_compiler in
+            1|true|yes|YES)
+                build_cmark=""
+                local_build=""
+                build_llvm="1"
+                build_swift_tools="1"
+                validation_test="1"
+                native_swift_tools_path=""
+                native_clang_tools_path=""
+                ;;
+            *)
+                build_cmark="--skip-build-cmark"
+                local_build="--skip-local-build"
+                build_llvm="0"
+                build_swift_tools="0"
+                validation_test="0"
+                native_swift_tools_path="--native-swift-tools-path=$host_toolchain/bin"
+                native_clang_tools_path="--native-clang-tools-path=$host_toolchain/bin"
+                ;;
+        esac
+
         # use an out-of-tree build folder, otherwise subsequent arch builds have conflicts
         export SWIFT_BUILD_ROOT=${build_dir}/$arch/swift-project
 
         ./swift/utils/build-script \
             $build_type_flag \
             --reconfigure \
             --no-assertions \
+            --validation-test=$validation_test \
             --android \
             --android-ndk=$ndk_home \
             --android-arch=$arch \
             --android-api-level=$android_api \
-            --native-swift-tools-path=$host_toolchain/bin \
-            --native-clang-tools-path=$host_toolchain/bin \
             --cross-compile-hosts=android-$arch \
             --cross-compile-deps-path=$sdk_root \
             --install-destdir=$sdk_root \
-            --build-llvm=0 \
-            --build-swift-tools=0 \
-            --skip-build-cmark \
-            --skip-local-build \
+            --build-llvm=$build_llvm \
+            --build-swift-tools=$build_swift_tools \
+            ${native_swift_tools_path} \
+            ${native_clang_tools_path} \
+            ${build_cmark} \
+            ${local_build} \
+            --host-test \
+            --skip-test-linux \
+            --skip-test-xctest \
             --build-swift-static-stdlib \
             --install-swift \
             --install-libdispatch \
             --install-foundation \
             --xctest --install-xctest \
-            --swift-testing --install-swift-testing \
-            --cross-compile-append-host-target-to-destdir=False \
-            --extra-cmake-options='-DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384"'
-
+            --cross-compile-build-swift-tools=0 \
+            --llvm-ninja-targets-for-cross-compile-hosts=help \
+            --cross-compile-append-host-target-to-destdir=False 
+            # --extra-cmake-options='-DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384"'
+        # --swift-testing --install-swift-testing \
         # need to remove symlink that gets created in the NDK to the previous arch's build
         # or else we get errors like:
         # error: could not find module '_Builtin_float' for target 'x86_64-unknown-linux-android'; found: aarch64-unknown-linux-android, at: /home/runner/work/_temp/swift-android-sdk/ndk/android-ndk-r27c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift/android/_Builtin_float.swiftmodule

swift-ci/sdks/android/scripts/fetch-source.sh

@@ -89,6 +89,9 @@ fi
 if [[ -z "${BORINGSSL_VERSION}" ]]; then
     BORINGSSL_VERSION=fips-20220613
 fi
+if [[ -z "${YAMS_VERSION}" ]]; then
+    YAMS_VERSION=5.0.6
+fi
 if [[ -z "${SWIFT_ANDROID_PATCHES_VERSION}" ]]; then
     SWIFT_ANDROID_PATCHES_VERSION=main
 fi
@@ -158,6 +161,16 @@ fi
 popd >/dev/null
 groupend
 
+# Fetch yams (needed for Swift 6.1.x)
+groupstart "Fetching yams"
+pushd swift-project >/dev/null
+[[ -d yams ]] || git clone ${github}jpsim/Yams.git yams
+pushd yams >/dev/null 2>&1
+git checkout ${YAMS_VERSION}
+popd >/dev/null 2>&1
+popd >/dev/null
+groupend
+
 # Fetch libxml2
 groupstart "Fetching libxml2"
 [[ -d libxml2 ]] || git clone ${github}GNOME/libxml2.git
@@ -182,7 +195,7 @@ git checkout ${BORINGSSL_VERSION}
 popd >/dev/null 2>&1
 groupend
 
-# Fetch BoringSSL
+# Fetch Patches
 groupstart "Fetching Patches"
 [[ -d swift-android-patches ]] || git clone https://github.com/swift-android-sdk/swift-android-sdk swift-android-patches
 pushd swift-android-patches >/dev/null 2>&1