Merge pull request #1398 from swiftwasm/maxd/5.3-build-host

Build both host and target toolchains in 5.3 branch

Author: MaxDesiatov

CMakeLists.txt

@@ -897,6 +897,36 @@ function(swift_icu_variables_set sdk arch result)
   endif()
 endfunction()
 
+if(SWIFT_PRIMARY_VARIANT_SDK STREQUAL "WASI")
+  set(WASI_ICU_URL "" CACHE STRING
+    "Download URL for the WASI ICU distribution")
+  set(WASI_ICU_MD5 "" CACHE STRING
+    "The expected MD5 hash of the WASI ICU distribution archive")
+
+  file(DOWNLOAD "${WASI_ICU_URL}" "${SWIFT_SOURCE_DIR}/../icu.tar.xz"
+    EXPECTED_HASH MD5=${WASI_ICU_MD5})
+
+  get_filename_component(SWIFT_SOURCE_BASEDIR "${SWIFT_SOURCE_DIR}" DIRECTORY)
+  set(WASI_ICU_OUT_LIB "${SWIFT_SOURCE_BASEDIR}/icu_out/lib")
+  set(SWIFT_WASI_wasm32_ICU_UC_INCLUDE "${SWIFT_SOURCE_BASEDIR}/icu_out/include")
+  set(SWIFT_WASI_wasm32_ICU_I18N_INCLUDE "${SWIFT_SOURCE_BASEDIR}/icu_out/include")
+  set(SWIFT_WASI_wasm32_ICU_UC "${WASI_ICU_OUT_LIB}/libicuuc.a")
+  set(SWIFT_WASI_wasm32_ICU_I18N "${WASI_ICU_OUT_LIB}/libicui18n.a")
+  set(SWIFT_WASI_wasm32_ICU_DATA "${WASI_ICU_OUT_LIB}/libicudata.a")
+
+  set(WASI_ICU_DISTRIBUTION_TARGET)
+  add_custom_command_target(WASI_ICU_DISTRIBUTION_TARGET
+    COMMAND 
+      ${CMAKE_COMMAND} -E 
+        tar xfv "${SWIFT_SOURCE_BASEDIR}/icu.tar.xz"
+    WORKING_DIRECTORY 
+      "${SWIFT_SOURCE_BASEDIR}"
+    OUTPUT
+      "${SWIFT_WASI_wasm32_ICU_DATA}"
+      "${SWIFT_WASI_wasm32_ICU_I18N}"
+      "${SWIFT_WASI_wasm32_ICU_UC}")
+endif()
+
 # ICU is provided through CoreFoundation on Darwin.  On other hosts, if the ICU
 # unicode and i18n include and library paths are not defined, perform a standard
 # package lookup.  Otherwise, rely on the paths specified by the user.  These

stdlib/public/stubs/CMakeLists.txt

@@ -27,7 +27,14 @@ set(swift_stubs_c_compile_flags ${SWIFT_RUNTIME_CORE_CXX_FLAGS})
 list(APPEND swift_stubs_c_compile_flags -DswiftCore_EXPORTS)
 list(APPEND swift_stubs_c_compile_flags -I${SWIFT_SOURCE_DIR}/include)
 
+set(swift_stubs_dependencies)
+if(SWIFT_PRIMARY_VARIANT_SDK STREQUAL "WASI")
+  list(APPEND swift_stubs_dependencies ${WASI_ICU_DISTRIBUTION_TARGET})
+endif()
+
 add_swift_target_library(swiftStdlibStubs
+                  DEPENDS
+                    ${swift_stubs_dependencies}
                   OBJECT_LIBRARY
                     ${swift_stubs_sources}
                     ${swift_stubs_objc_sources}

utils/build-presets.ini

@@ -2414,3 +2414,98 @@ no-assertions
 mixin-preset=source_compat_suite_linux_base
 debug
 no-assertions
+
+#===----------------------------------------------------------------------===#
+# Preset for the WebAssembly toolchain and SDK
+#===----------------------------------------------------------------------===#
+
+[preset: webassembly]
+
+release
+cmake-c-launcher=%(C_CXX_LAUNCHER)s
+cmake-cxx-launcher=%(C_CXX_LAUNCHER)s
+skip-build-benchmarks
+llvm-targets-to-build=X86;WebAssembly
+install-destdir=%(INSTALL_DESTDIR)s
+swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
+llvm-install-components=clang
+install-swift
+install-prefix=/%(TOOLCHAIN_NAME)s/usr
+
+[preset: webassembly-host]
+
+mixin-preset=webassembly
+extra-cmake-options=
+    -DSWIFT_BUILD_SOURCEKIT=FALSE
+    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
+    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
+
+llbuild
+swiftpm
+
+install-llvm
+install-swift
+install-llbuild
+install-swiftpm
+
+[preset: webassembly-linux-host]
+
+mixin-preset=webassembly-host
+
+libdispatch
+libicu
+foundation
+xctest
+
+install-libicu
+install-foundation
+install-libdispatch
+install-xctest
+
+skip-test-libicu
+skip-test-foundation
+skip-test-libdispatch
+skip-test-xctest
+
+[preset: webassembly-target]
+
+mixin-preset=webassembly
+wasm
+verbose
+build-stdlib-deployment-targets=wasi-wasm32
+build-swift-dynamic-sdk-overlay=false
+build-swift-dynamic-stdlib=false
+build-swift-static-sdk-overlay
+build-swift-static-stdlib
+stdlib-deployment-targets=wasi-wasm32
+wasi-sdk=%(SOURCE_PATH)s/wasi-sdk
+
+[preset: webassembly-linux-target]
+
+mixin-preset=webassembly-target
+extra-cmake-options=
+    -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
+    -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
+    -DSWIFT_PRIMARY_VARIANT_SDK:STRING=WASI
+    -DSWIFT_PRIMARY_VARIANT_ARCH:STRING=wasm32
+    -DSWIFT_SDKS='WASI;LINUX'
+    -DSWIFT_BUILD_SOURCEKIT=FALSE
+    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
+    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
+    -DCMAKE_AR="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ar"
+    -DCMAKE_RANLIB="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ranlib"
+
+[preset: webassembly-macos-target]
+
+mixin-preset=webassembly-target
+extra-cmake-options=
+    -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
+    -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
+    -DSWIFT_PRIMARY_VARIANT_SDK:STRING=WASI
+    -DSWIFT_PRIMARY_VARIANT_ARCH:STRING=wasm32
+    -DSWIFT_OSX_x86_64_ICU_STATICLIB=TRUE
+    -DSWIFT_BUILD_SOURCEKIT=FALSE
+    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
+    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
+    -DCMAKE_AR='/usr/local/opt/llvm/bin/llvm-ar'
+    -DCMAKE_RANLIB='/usr/local/opt/llvm/bin/llvm-ranlib'

utils/build-script

@@ -215,18 +215,9 @@ def validate_arguments(toolchain, args):
                 "--android-icu-i18n-include, and --android-icu-data "
                 "must be specified")
     if args.wasm:
-        if args.wasi_sdk is None or \
-                args.wasi_icu_uc is None or \
-                args.wasi_icu_uc_include is None or \
-                args.wasi_icu_i18n is None or \
-                args.wasi_icu_i18n_include is None or \
-                args.wasi_icu_data is None:
-            diagnostics.fatal(
-                "when building for WebAssembly, --wasi-sdk, "
-                "--wasi-icu-uc, "
-                "--wasi-icu-uc-include, --wasi-icu-i18n, "
-                "--wasi-icu-i18n-include, and --wasi-icu-data "
-                "must be specified")
+        if args.wasi_sdk is None:
+            fatal_error(
+                "when building for WebAssembly, --wasi-sdk must be specified")
 
     targets_needing_toolchain = [
         'build_indexstoredb',
@@ -708,11 +699,6 @@ class BuildScriptInvocation(object):
         if args.wasm:
             impl_args += [
                 "--wasi-sdk", args.wasi_sdk,
-                "--wasi-icu-uc", args.wasi_icu_uc,
-                "--wasi-icu-uc-include", args.wasi_icu_uc_include,
-                "--wasi-icu-i18n", args.wasi_icu_i18n,
-                "--wasi-icu-i18n-include", args.wasi_icu_i18n_include,
-                "--wasi-icu-data", args.wasi_icu_data,
             ]
 
         if platform.system() == 'Darwin':

utils/build-script-impl

@@ -118,11 +118,6 @@ KNOWN_SETTINGS=(
 
     ## WebAssembly/WASI Options
     wasi-sdk                                      ""                "An absolute path to the WASI SDK that will be used as a libc implementation for Wasm builds"
-    wasi-icu-uc                                   ""                "Path to libicuuc.so"
-    wasi-icu-uc-include                           ""                "Path to a directory containing headers for libicuuc"
-    wasi-icu-i18n                                 ""                "Path to libicui18n.so"
-    wasi-icu-i18n-include                         ""                "Path to a directory containing headers libicui18n"
-    wasi-icu-data                                 ""                "Path to libicudata.so"
 
     ## Build Types for Components
     swift-stdlib-build-type                       "Debug"           "the CMake build variant for Swift"
@@ -1694,11 +1689,6 @@ for host in "${ALL_HOSTS[@]}"; do
                     cmake_options=(
                         "${cmake_options[@]}"
                         -DSWIFT_WASI_SDK_PATH:STRING="${WASI_SDK}"
-                        -DSWIFT_WASI_wasm32_ICU_UC:STRING="${WASI_ICU_UC}"
-                        -DSWIFT_WASI_wasm32_ICU_UC_INCLUDE:STRING="${WASI_ICU_UC_INCLUDE}"
-                        -DSWIFT_WASI_wasm32_ICU_I18N:STRING="${WASI_ICU_I18N}"
-                        -DSWIFT_WASI_wasm32_ICU_I18N_INCLUDE:STRING="${WASI_ICU_I18N_INCLUDE}"
-                        -DSWIFT_WASI_wasm32_ICU_DATA:STRING="${WASI_ICU_DATA}"
                     )
                 fi
 

utils/build_swift/build_swift/driver_arguments.py

@@ -1095,17 +1095,6 @@ def create_argument_parser():
            help='An absolute path to WASI SDK that will be used as a libc '
                 'implementation for Wasm builds')
 
-    option('--wasi-icu-uc', store_path,
-           help='Path to libicuuc.so')
-    option('--wasi-icu-uc-include', store_path,
-           help='Path to a directory containing headers for libicuuc')
-    option('--wasi-icu-i18n', store_path,
-           help='Path to libicui18n.so')
-    option('--wasi-icu-i18n-include', store_path,
-           help='Path to a directory containing headers libicui18n')
-    option('--wasi-icu-data', store_path,
-           help='Path to libicudata.so')
-
     # -------------------------------------------------------------------------
     in_group('Experimental language features')
 

utils/update_checkout/update-checkout-config.json

@@ -59,6 +59,7 @@
                 "swift": "swiftwasm",
                 "cmark": "master",
                 "llbuild": "master",
+                "swift-tools-support-core": "master",
                 "swiftpm": "swiftwasm",
                 "swift-syntax": "master",
                 "swift-stress-tester": "master",
@@ -82,6 +83,7 @@
                 "swift": "swiftwasm-release/5.3",
                 "cmark": "release/5.3",
                 "llbuild": "release/5.3",
+                "swift-tools-support-core": "release/5.3",
                 "swiftpm": "swiftwasm-release/5.3",
                 "swift-syntax": "release/5.3",
                 "swift-stress-tester": "release/5.3",

utils/webassembly/build-foundation.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -ex
+DESTINATION_TOOLCHAIN=$1
+SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
+
+# Remove host CoreFoundation (which can be different from the target) headers
+# to avoid shadowing the wasm32 target CoreFoundation
+rm -rf $DESTINATION_TOOLCHAIN/usr/lib/swift/CoreFoundation
+
+FOUNDATION_BUILD="$SOURCE_PATH/build/Ninja-ReleaseAssert/foundation-wasi-wasm32"
+
+mkdir -p $FOUNDATION_BUILD
+cd $FOUNDATION_BUILD
+
+cmake -G Ninja \
+  -DCMAKE_Swift_COMPILER="$DESTINATION_TOOLCHAIN/usr/bin/swiftc" \
+  -DCMAKE_STAGING_PREFIX="$DESTINATION_TOOLCHAIN/usr" \
+  -DCMAKE_TOOLCHAIN_FILE="$SOURCE_PATH/swift/utils/webassembly/toolchain-wasi.cmake" \
+  -DWASI_SDK_PATH="$SOURCE_PATH/wasi-sdk" \
+  -DICU_ROOT="$SOURCE_PATH/icu_out" \
+  -DBUILD_SHARED_LIBS=OFF \
+  "${SOURCE_PATH}/swift-corelibs-foundation"
+  
+ninja -v
+ninja -v install
+
+# On macOS the target CoreFoundation shadows the CoreFoundation suppplied by Xcode.
+# On Linux though there's no system CoreFoundation, its headers have to be shipped
+# in the installable archive and serve for both host and target.
+if [[ "$(uname)" == "Darwin" ]]; then
+  mv $DESTINATION_TOOLCHAIN/usr/lib/swift/CoreFoundation \
+    $DESTINATION_TOOLCHAIN/usr/lib/swift/wasi/wasm32/CoreFoundation
+fi