Merge pull request #2041 from swiftwasm/maxd/5.3-cherry-pick

MaxDesiatov · web-flow · commit 75ee0115dec2 · 2020-10-20T16:56:55.000+01:00
Cherry-pick packaging changes for the 5.3 branch
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -51,7 +51,7 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: ubuntu18.04-installable
-          path: ../swift-wasm-5.3-SNAPSHOT-ubuntu18.04-x86_64.tar.gz
+          path: ../swift-wasm-5.3-SNAPSHOT-ubuntu18.04_x86_64.tar.gz
 #      - name: Pack test results
 #        run: tar cJf swift-test-results.tar.gz ../build/*/swift-linux-x86_64/swift-test-results
 #      - name: Upload test results
@@ -98,7 +98,7 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: ubuntu20.04-installable
-          path: ../swift-wasm-5.3-SNAPSHOT-ubuntu20.04-x86_64.tar.gz
+          path: ../swift-wasm-5.3-SNAPSHOT-ubuntu20.04_x86_64.tar.gz
 #      - name: Pack test results
 #        run: tar cJf swift-test-results.tar.gz ../build/*/swift-linux-x86_64/swift-test-results
 #      - name: Upload test results
@@ -135,7 +135,7 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: macos-installable
-          path: ../swift-wasm-5.3-SNAPSHOT-macos-x86_64.tar.gz
+          path: ../swift-wasm-5.3-SNAPSHOT-macos_x86_64.tar.gz
       - name: Pack test results
         run: tar cJf swift-test-results.tar.gz ../target-build/*/swift-macosx-x86_64/swift-test-results
       - name: Upload test results
diff --git a/.github/workflows/nightly-distribution.yml b/.github/workflows/nightly-distribution.yml
@@ -7,11 +7,32 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v1
+      - name: Setup keychain
+        run: |
+          echo "$DARWIN_TOOLCHAIN_APPLICATION_CERT_BASE64" | base64 --decode -o developerID_application.p12
+          security import developerID_application.p12 -P "$DARWIN_TOOLCHAIN_APPLICATION_CERT_PASSWORD"
+          echo "$DARWIN_TOOLCHAIN_INSTALLER_CERT_BASE64" | base64 --decode -o developerID_installer.p12
+          security import developerID_installer.p12 -P "$DARWIN_TOOLCHAIN_INSTALLER_CERT_PASSWORD"
+        env:
+          DARWIN_TOOLCHAIN_APPLICATION_CERT: ${{ secrets.DARWIN_TOOLCHAIN_APPLICATION_CERT }}
+          DARWIN_TOOLCHAIN_APPLICATION_CERT_BASE64: ${{ secrets.DARWIN_TOOLCHAIN_APPLICATION_CERT_BASE64 }}
+          DARWIN_TOOLCHAIN_APPLICATION_CERT_PASSWORD: ${{ secrets.DARWIN_TOOLCHAIN_APPLICATION_CERT_PASSWORD }}
+          DARWIN_TOOLCHAIN_INSTALLER_CERT: ${{ secrets.DARWIN_TOOLCHAIN_INSTALLER_CERT }}
+          DARWIN_TOOLCHAIN_INSTALLER_CERT_BASE64: ${{ secrets.DARWIN_TOOLCHAIN_INSTALLER_CERT_BASE64 }}
+          DARWIN_TOOLCHAIN_INSTALLER_CERT_PASSWORD: ${{ secrets.DARWIN_TOOLCHAIN_INSTALLER_CERT_PASSWORD }}
       - run: ./utils/webassembly/distribute-latest-toolchain.sh swiftwasm DEVELOPMENT
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DARWIN_TOOLCHAIN_APPLICATION_CERT: ${{ secrets.DARWIN_TOOLCHAIN_APPLICATION_CERT }}
+          DARWIN_TOOLCHAIN_INSTALLER_CERT: ${{ secrets.DARWIN_TOOLCHAIN_INSTALLER_CERT }}
+          DARWIN_TOOLCHAIN_NOTARIZE_EMAIL: ${{ secrets.DARWIN_TOOLCHAIN_NOTARIZE_EMAIL }}
+          DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD: ${{ secrets.DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD }}
         if: github.ref == 'refs/heads/swiftwasm'
       - run: ./utils/webassembly/distribute-latest-toolchain.sh swiftwasm-release/5.3 5.3
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DARWIN_TOOLCHAIN_APPLICATION_CERT: ${{ secrets.DARWIN_TOOLCHAIN_APPLICATION_CERT }}
+          DARWIN_TOOLCHAIN_INSTALLER_CERT: ${{ secrets.DARWIN_TOOLCHAIN_INSTALLER_CERT }}
+          DARWIN_TOOLCHAIN_NOTARIZE_EMAIL: ${{ secrets.DARWIN_TOOLCHAIN_NOTARIZE_EMAIL }}
+          DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD: ${{ secrets.DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD }}
         if: github.ref == 'refs/heads/swiftwasm-release/5.3'
diff --git a/utils/webassembly/build-toolchain.sh b/utils/webassembly/build-toolchain.sh
@@ -8,16 +8,16 @@ WASI_SDK_PATH=$SOURCE_PATH/wasi-sdk
 
 case $(uname -s) in
   Darwin)
-    OS_SUFFIX=macos-x86_64
+    OS_SUFFIX=macos_x86_64
     HOST_PRESET=webassembly-host
     TARGET_PRESET=webassembly-macos-target
     HOST_SUFFIX=macosx-x86_64
   ;;
   Linux)
     if [ $(grep RELEASE /etc/lsb-release) == "DISTRIB_RELEASE=18.04" ]; then
-      OS_SUFFIX=ubuntu18.04-x86_64
+      OS_SUFFIX=ubuntu18.04_x86_64
     elif [ $(grep RELEASE /etc/lsb-release) == "DISTRIB_RELEASE=20.04" ]; then
-      OS_SUFFIX=ubuntu20.04-x86_64
+      OS_SUFFIX=ubuntu20.04_x86_64
     else
       echo "Unknown Ubuntu version"
       exit 1
@@ -124,11 +124,55 @@ merge_toolchains() {
   sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $DIST_TOOLCHAIN_SDK/usr/lib/swift/wasi/wasm32/wasi.modulemap
 }
 
+create_darwin_info_plist() {
+  echo "-- Create Info.plist --"
+  PLISTBUDDY_BIN="/usr/libexec/PlistBuddy"
+
+  DARWIN_TOOLCHAIN_VERSION="5.3.${YEAR}${MONTH}${DAY}"
+  BUNDLE_PREFIX="org.swiftwasm"
+  DARWIN_TOOLCHAIN_BUNDLE_IDENTIFIER="${BUNDLE_PREFIX}.${YEAR}${MONTH}${DAY}"
+  DARWIN_TOOLCHAIN_DISPLAY_NAME_SHORT="Swift for WebAssembly Snapshot"
+  DARWIN_TOOLCHAIN_DISPLAY_NAME="${DARWIN_TOOLCHAIN_DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
+  DARWIN_TOOLCHAIN_ALIAS="swiftwasm"
+
+  DARWIN_TOOLCHAIN_INFO_PLIST="${DIST_TOOLCHAIN_SDK}/usr/Info.plist"
+  DARWIN_TOOLCHAIN_REPORT_URL="https://bugs.swift.org/"
+  COMPATIBILITY_VERSION=2
+  COMPATIBILITY_VERSION_DISPLAY_STRING="Xcode 8.0"
+  DARWIN_TOOLCHAIN_CREATED_DATE="$(date -u +'%a %b %d %T GMT %Y')"
+  SWIFT_USE_DEVELOPMENT_TOOLCHAIN_RUNTIME="YES"
+
+  rm -f "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+
+  ${PLISTBUDDY_BIN} -c "Add DisplayName string '${DARWIN_TOOLCHAIN_DISPLAY_NAME}'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add ShortDisplayName string '${DARWIN_TOOLCHAIN_DISPLAY_NAME_SHORT}'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add CreatedDate date '${DARWIN_TOOLCHAIN_CREATED_DATE}'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add CompatibilityVersion integer ${COMPATIBILITY_VERSION}" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add CompatibilityVersionDisplayString string ${COMPATIBILITY_VERSION_DISPLAY_STRING}" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add Version string '${DARWIN_TOOLCHAIN_VERSION}'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add CFBundleIdentifier string '${DARWIN_TOOLCHAIN_BUNDLE_IDENTIFIER}'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add ReportProblemURL string '${DARWIN_TOOLCHAIN_REPORT_URL}'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add Aliases array" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add Aliases:0 string '${DARWIN_TOOLCHAIN_ALIAS}'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add OverrideBuildSettings dict" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add OverrideBuildSettings:ENABLE_BITCODE string 'NO'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add OverrideBuildSettings:SWIFT_DISABLE_REQUIRED_ARCLITE string 'YES'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add OverrideBuildSettings:SWIFT_LINK_OBJC_RUNTIME string 'YES'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add OverrideBuildSettings:SWIFT_DEVELOPMENT_TOOLCHAIN string 'YES'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+  ${PLISTBUDDY_BIN} -c "Add OverrideBuildSettings:SWIFT_USE_DEVELOPMENT_TOOLCHAIN_RUNTIME string '${SWIFT_USE_DEVELOPMENT_TOOLCHAIN_RUNTIME}'" "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+
+  chmod a+r "${DARWIN_TOOLCHAIN_INFO_PLIST}"
+}
+
 build_host_toolchain
 build_target_toolchain
 
 merge_toolchains
 
-cd $DIST_TOOLCHAIN_DESTDIR
-tar cfz $PACKAGE_ARTIFACT $TOOLCHAIN_NAME
+if [[ "$(uname)" == "Darwin" ]]; then
+  create_darwin_info_plist
+fi
+
+cd "$DIST_TOOLCHAIN_DESTDIR"
+tar cfz "$PACKAGE_ARTIFACT" "$TOOLCHAIN_NAME"
 echo "Toolchain archive created successfully!"
diff --git a/utils/webassembly/distribute-latest-toolchain.sh b/utils/webassembly/distribute-latest-toolchain.sh
@@ -4,6 +4,12 @@ set -xe
 repository='swiftwasm/swift'
 workflow_name='main.yml'
 branch=$1
+channel=$2
+swift_source_dir="$(cd "$(dirname $0)/../.." && pwd)"
+
+DARWIN_TOOLCHAIN_APPLICATION_CERT=${DARWIN_TOOLCHAIN_APPLICATION_CERT:?"Please set DARWIN_TOOLCHAIN_APPLICATION_CERT"}
+DARWIN_TOOLCHAIN_INSTALLER_CERT=${DARWIN_TOOLCHAIN_INSTALLER_CERT:?"Please set DARWIN_TOOLCHAIN_APPLICATION_CERT"}
+DARWIN_TOOLCHAIN_NOTARIZE_EMAIL=${DARWIN_TOOLCHAIN_NOTARIZE_EMAIL:?"Please set DARWIN_TOOLCHAIN_NOTARIZE_EMAIL"}
 
 gh_api=https://api.github.com
 
@@ -95,6 +101,72 @@ upload_tarball() {
     "https://uploads.github.com/repos/$repository/releases/$release_id/assets?name=$filename"
 }
 
+sign_toolchain() {
+  local darwin_toolchain=$1
+  local codesign_bin="/usr/bin/codesign"
+
+  codesign_args=(--force --verify --verbose --deep --options runtime --timestamp --sign "${DARWIN_TOOLCHAIN_APPLICATION_CERT}")
+  for binary in $(find "${darwin_toolchain}" -type f); do
+    if file "$binary" | grep -q "Mach-O"; then
+        ${codesign_bin} "${codesign_args[@]}" "${binary}"
+    fi
+  done
+
+  ${codesign_bin} "${codesign_args[@]}" "${darwin_toolchain}/usr/"
+}
+
+create_installer() {
+  local darwin_toolchain=$1
+  local darwin_toolchain_name=$(basename "$darwin_toolchain")
+  local darwin_toolchain_installer_package="$darwin_toolchain.pkg"
+  local darwin_toolchain_install_location="/Library/Developer/Toolchains/${darwin_toolchain_name}.xctoolchain"
+  local darwin_toolchain_version=$(/usr/libexec/PlistBuddy  -c "Print Version string" "$darwin_toolchain"/usr/Info.plist)
+  local darwin_toolchain_bundle_identifier=$(/usr/libexec/PlistBuddy  -c "Print CFBundleIdentifier string" "$darwin_toolchain"/usr/Info.plist)
+
+  "${swift_source_dir}/utils/toolchain-installer" "${darwin_toolchain}/" "${darwin_toolchain_bundle_identifier}" \
+    "${DARWIN_TOOLCHAIN_INSTALLER_CERT}" "${darwin_toolchain_installer_package}" "${darwin_toolchain_install_location}" \
+    "${darwin_toolchain_version}" "${swift_source_dir}/utils/darwin-installer-scripts"
+
+  # Notarize the toolchain installer
+  local request_output=$(xcrun altool --notarize-app --type osx \
+      --file "${darwin_toolchain_installer_package}" \
+      --primary-bundle-id "${darwin_toolchain_bundle_identifier}" \
+      -u "${DARWIN_TOOLCHAIN_NOTARIZE_EMAIL}" \
+      -p "@env:DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD")
+  local request_uuid=$(echo "$request_output" | grep "RequestUUID = " | awk '{print $3}')
+
+  local request_status=$(xcrun altool --notarization-info "$request_uuid" \
+      -u "${DARWIN_TOOLCHAIN_NOTARIZE_EMAIL}" \
+      -p "@env:DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD")
+  # Wait until finished
+  while echo "$request_status" | grep -q "Status: in progress" ; do
+    sleep 60
+    request_status=$(xcrun altool --notarization-info "$request_uuid" \
+      -u "${DARWIN_TOOLCHAIN_NOTARIZE_EMAIL}" \
+      -p "@env:DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD")
+  done
+
+  if echo "$request_status" | grep -q "Status: success"; then
+    xcrun stapler staple "${darwin_toolchain_installer_package}"
+  else
+    echo "Failed to notarize the toolchain $darwin_toolchain_installer_package: $request_status"
+  fi
+}
+
+package_darwin_toolchain() {
+  local toolchain_tar=$1
+  local destination=$2
+  local toolchain_name=$(basename $(tar tfz "$toolchain_tar" | head -n1))
+  local workdir=$(mktemp -d)
+
+  tar xfz "$toolchain_tar" -C "$workdir"
+  sign_toolchain "$workdir/$toolchain_name"
+  create_installer "$workdir/$toolchain_name"
+
+  mv "$workdir/$toolchain_name.pkg" "$destination"
+  rm -rf "$workdir"
+}
+
 tmp_dir=$(mktemp -d)
 pushd $tmp_dir
 download_artifact ubuntu18.04-installable
@@ -104,22 +176,23 @@ unzip ubuntu18.04-installable.zip
 unzip ubuntu20.04-installable.zip
 unzip macos-installable.zip
 
-toolchain_name=$(basename $(tar tfz swift-wasm-$2-SNAPSHOT-ubuntu18.04-x86_64.tar.gz | head -n1))
+toolchain_name=$(basename $(tar tfz swift-wasm-$channel-SNAPSHOT-ubuntu18.04_x86_64.tar.gz | head -n1))
 
 if is_released $toolchain_name; then
   echo "Latest toolchain $toolchain_name has been already released"
   exit 0
 fi
 
-mv swift-wasm-$2-SNAPSHOT-ubuntu18.04-x86_64.tar.gz "$toolchain_name-ubuntu18.04-x86_64.tar.gz"
-mv swift-wasm-$2-SNAPSHOT-ubuntu20.04-x86_64.tar.gz "$toolchain_name-ubuntu20.04-x86_64.tar.gz"
-mv swift-wasm-$2-SNAPSHOT-macos-x86_64.tar.gz "$toolchain_name-macos-x86_64.tar.gz"
+
+mv swift-wasm-$channel-SNAPSHOT-ubuntu18.04_x86_64.tar.gz "$toolchain_name-ubuntu18.04_x86_64.tar.gz"
+mv swift-wasm-$channel-SNAPSHOT-ubuntu20.04_x86_64.tar.gz "$toolchain_name-ubuntu20.04_x86_64.tar.gz"
+package_darwin_toolchain "swift-wasm-$channel-SNAPSHOT-macos_x86_64.tar.gz" "$toolchain_name-macos_x86_64.pkg"
 
 create_tag $toolchain_name $head_sha
 release_id=$(create_release $toolchain_name $toolchain_name $head_sha)
 
-upload_tarball $release_id "$toolchain_name-ubuntu18.04-x86_64.tar.gz"
-upload_tarball $release_id "$toolchain_name-ubuntu20.04-x86_64.tar.gz"
-upload_tarball $release_id "$toolchain_name-macos-x86_64.tar.gz"
+upload_tarball $release_id "$toolchain_name-ubuntu18.04_x86_64.tar.gz"
+upload_tarball $release_id "$toolchain_name-ubuntu20.04_x86_64.tar.gz"
+upload_tarball $release_id "$toolchain_name-macos_x86_64.pkg"
 
 popd
