Merge pull request #1387 from swiftwasm/maxd/build-host

Build both host and target toolchains on CI

Both `libPackageDescription.dylib` and `swiftc` used to compile package manifests should come from the same toolchain for SwiftPM to work on macOS. This is resolved here by re-reverting the previous change #1024 to build both host and target toolchain in one go on CI. I'm wary of the disk space/time limit constraints, but I think those should be resolved separately when they appear, otherwise SwiftPM is completely broken on macOS currently. Resolves #1365.

Author: MaxDesiatov

utils/build-presets.ini

@@ -2461,40 +2461,68 @@ no-assertions
 
 [preset: webassembly]
 
-wasm
 release
-verbose
 cmake-c-launcher=%(C_CXX_LAUNCHER)s
 cmake-cxx-launcher=%(C_CXX_LAUNCHER)s
 skip-build-benchmarks
+llvm-targets-to-build=X86;WebAssembly
+install-destdir=%(INSTALL_DESTDIR)s
+swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
+llvm-install-components=clang
+install-swift
+install-prefix=/%(TOOLCHAIN_NAME)s/usr
+
+[preset: webassembly-host]
+
+mixin-preset=webassembly
+extra-cmake-options=
+    -DSWIFT_BUILD_SOURCEKIT=FALSE
+    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
+    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
+
+llbuild
+swiftpm
+
+install-llvm
+install-swift
+install-llbuild
+install-swiftpm
+
+[preset: webassembly-linux-host]
+
+mixin-preset=webassembly-host
+
+libdispatch
+libicu
+foundation
+xctest
+
+install-libicu
+install-foundation
+install-libdispatch
+install-xctest
+
+skip-test-libicu
+skip-test-foundation
+skip-test-libdispatch
+skip-test-xctest
+
+[preset: webassembly-target]
+
+mixin-preset=webassembly
+wasm
+verbose
 build-stdlib-deployment-targets=wasi-wasm32
 build-swift-dynamic-sdk-overlay=false
 build-swift-dynamic-stdlib=false
 build-swift-static-sdk-overlay
 build-swift-static-stdlib
-llvm-targets-to-build=X86;WebAssembly
 stdlib-deployment-targets=wasi-wasm32
 wasi-sdk=%(SOURCE_PATH)s/wasi-sdk
 
-[preset: webassembly-installable]
-
-install-destdir=%(SOURCE_PATH)s/install
-installable-package=%(INSTALLABLE_PACKAGE)s
-install-prefix=/%(TOOLCHAIN_NAME)s/usr
-swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
-llvm-install-components=clang
-install-swift
-install-llvm
-darwin-toolchain-bundle-identifier=%(BUNDLE_IDENTIFIER)s
-darwin-toolchain-display-name=%(DISPLAY_NAME)s
-darwin-toolchain-display-name-short=%(DISPLAY_NAME_SHORT)s
-darwin-toolchain-name=%(TOOLCHAIN_NAME)s
-darwin-toolchain-version=%(TOOLCHAIN_VERSION)s
-darwin-toolchain-alias=swift
+[preset: webassembly-linux-target]
 
-[preset: webassembly-linux]
-
-mixin-preset=webassembly
+mixin-preset=webassembly-target
 extra-cmake-options=
     -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
     -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
@@ -2507,9 +2535,9 @@ extra-cmake-options=
     -DCMAKE_AR="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ar"
     -DCMAKE_RANLIB="%(SOURCE_PATH)s/wasi-sdk/bin/llvm-ranlib"
 
-[preset: webassembly-macos]
+[preset: webassembly-macos-target]
 
-mixin-preset=webassembly
+mixin-preset=webassembly-target
 extra-cmake-options=
     -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
     -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
@@ -2521,15 +2549,3 @@ extra-cmake-options=
     -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
     -DCMAKE_AR='/usr/local/opt/llvm/bin/llvm-ar'
     -DCMAKE_RANLIB='/usr/local/opt/llvm/bin/llvm-ranlib'
-
-[preset: webassembly-linux-installable]
-
-mixin-preset=
-    webassembly-linux
-    webassembly-installable
-
-[preset: webassembly-macos-installable]
-
-mixin-preset=
-    webassembly-macos
-    webassembly-installable

utils/webassembly/build-swiftpm.sh

@@ -9,11 +9,13 @@ WASI_SDK_PATH=$SOURCE_PATH/wasi-sdk
 case $(uname -s) in
   Darwin)
     OS_SUFFIX=osx
-    PRESET_NAME=webassembly-macos-installable
+    HOST_PRESET=webassembly-host
+    TARGET_PRESET=webassembly-macos-target
   ;;
   Linux)
     OS_SUFFIX=linux
-    PRESET_NAME=webassembly-linux-installable
+    HOST_PRESET=webassembly-linux-host
+    TARGET_PRESET=webassembly-linux-target
   ;;
   *)
     echo "Unrecognised platform $(uname -s)"
@@ -35,49 +37,52 @@ BUNDLE_IDENTIFIER="swiftwasm.${YEAR}${MONTH}${DAY}"
 DISPLAY_NAME_SHORT="Swift for WebAssembly Development Snapshot"
 DISPLAY_NAME="${DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
 
-# Make sure Clang headers install dir exists to avoid broken symlinks
-mkdir -p $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/clang/10.0.0
+HOST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/host-toolchain-sdk
+HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
 
-$SOURCE_PATH/swift/utils/build-script --preset=$PRESET_NAME \
+# Avoid clang headers symlink issues
+mkdir -p $HOST_TOOLCHAIN_SDK/usr/lib/clang/10.0.0
+
+# Build the host toolchain and SDK first.
+$SOURCE_PATH/swift/utils/build-script \
+  --preset=$HOST_PRESET \
+  INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
+  TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
+  C_CXX_LAUNCHER="$(which sccache)"
+
+# Clean up the host toolchain build directory so that the next
+# `build-script` invocation doesn't pick up wrong CMake config files.
+# For some reason passing `--reconfigure` to `build-script` won't do this.
+rm -rf $SOURCE_PATH/build/Ninja-ReleaseAssert/swift-*
+
+# build the cross-compilled toolchain
+$SOURCE_PATH/swift/utils/build-script \
+  --preset=$TARGET_PRESET \
+  INSTALL_DESTDIR="$SOURCE_PATH/install" \
   SOURCE_PATH="$SOURCE_PATH" \
-  INSTALLABLE_PACKAGE="$INSTALLABLE_PACKAGE" \
   BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
   DISPLAY_NAME="${DISPLAY_NAME}" \
   DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
   TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
   TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
   C_CXX_LAUNCHER="$(which sccache)"
 
-
-NIGHTLY_TOOLCHAIN=$SOURCE_PATH/swift-nightly-toolchain
-if [ ! -e $NIGHTLY_TOOLCHAIN ]; then
-  $UTILS_PATH/install-nightly-toolchain.sh
-fi
-
-TMP_DIR=$(mktemp -d)
-cd $TMP_DIR
-tar xfz $INSTALLABLE_PACKAGE $TOOLCHAIN_NAME
-cd $TMP_DIR/$TOOLCHAIN_NAME
-
-# Merge wasi-sdk and toolchain
-cp -r $WASI_SDK_PATH/lib/clang usr/lib
-cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} usr/bin
-cp -r $WASI_SDK_PATH/share/wasi-sysroot usr/share
-
-# Build SwiftPM and install it into toolchain
-$UTILS_PATH/build-swiftpm.sh $TMP_DIR/$TOOLCHAIN_NAME
+# Merge wasi-sdk and the toolchain
+cp -a $WASI_SDK_PATH/lib/clang $HOST_TOOLCHAIN_SDK/usr/lib
+cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} $HOST_TOOLCHAIN_SDK/usr/bin
+cp -r $WASI_SDK_PATH/share/wasi-sysroot $HOST_TOOLCHAIN_SDK/usr/share
 
 # Replace absolute sysroot path with relative path
-sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
+sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
 
-# Copy nightly-toolchain's host environment stdlib into toolchain
+# Copy the target environment stdlib into the toolchain
 
 # Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
-# but nightly's one is symbolic link, simple copy fails to merge them.
-rsync -a $NIGHTLY_TOOLCHAIN/usr/lib/ $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/ --exclude 'swift/clang'
+# but nightly's one is symbolic link. A simple copy fails to merge them.
+rsync -v -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/ $HOST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
 
-$UTILS_PATH/build-foundation.sh $TMP_DIR/$TOOLCHAIN_NAME
-$UTILS_PATH/build-xctest.sh $TMP_DIR/$TOOLCHAIN_NAME
+$UTILS_PATH/build-foundation.sh $HOST_TOOLCHAIN_SDK
+$UTILS_PATH/build-xctest.sh $HOST_TOOLCHAIN_SDK
 
-cd $TMP_DIR
+cd $HOST_TOOLCHAIN_DESTDIR
 tar cfz $PACKAGE_ARTIFACT $TOOLCHAIN_NAME