From 9067da04689d27a4ebf071fa437d7a89d07d914b Mon Sep 17 00:00:00 2001 From: Norbert Lange Date: Tue, 16 Jan 2024 00:49:03 +0100 Subject: [PATCH 1/3] improve tar packaging The generated tar archives contain the uid/gid of the builder, if unpacked as root on another system those will denote nothing or an unexpected user/group. The expectation would be uid/gid = 0. Use a simple enough tar format as well. The Mac version of tar (bsdtar?) does not support those flags, for CI the separate gnu-tar is installed. --- .github/workflows/build.yml | 12 ++++++------ .github/workflows/msvcrt.yml | 2 +- release-macos.sh | 2 +- release.sh | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ba12fd6..b437564c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,7 +77,7 @@ jobs: DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-$(uname -m) NAME=llvm-mingw-$TAG-ucrt-$DISTRO mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME - uses: actions/upload-artifact@v4 with: name: linux-ucrt-x86_64-toolchain @@ -124,7 +124,7 @@ jobs: DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-aarch64 NAME=llvm-mingw-$TAG-ucrt-$DISTRO mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME - uses: actions/upload-artifact@v4 with: name: linux-ucrt-aarch64-toolchain @@ -157,7 +157,7 @@ jobs: LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLVM_ENABLE_TERMINFO=OFF" ./build-all.sh $(pwd)/install/llvm-mingw --disable-clang-tools-extra --disable-lldb --enable-asserts .github/workflows/store-version.sh install/llvm-mingw/versions.txt cd install - tar -Jcf ../llvm-mingw-linux.tar.xz llvm-mingw + tar -Jcf ../llvm-mingw-linux.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 llvm-mingw - uses: actions/upload-artifact@v4 with: name: linux-asserts-toolchain @@ -178,7 +178,7 @@ jobs: MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}} TAG: ${{needs.prepare.outputs.TAG}} run: | - brew install ninja + brew install ninja gnu-tar # Disable zstd and python. Both are available on the runners, but # installed with homebrew, and only available in the native (x86_64) # form. Therefore, autodetection will pick them up, but linking @@ -189,7 +189,7 @@ jobs: cd install NAME=llvm-mingw-$TAG-ucrt-macos-universal mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz $NAME + gtar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME - uses: actions/upload-artifact@v4 with: name: macos-ucrt-toolchain @@ -241,7 +241,7 @@ jobs: cd install NAME=llvm-mingw-$TAG-ucrt-msys2-${{matrix.sys}} mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME - uses: actions/upload-artifact@v4 with: name: msys2-${{matrix.sys}}-toolchain diff --git a/.github/workflows/msvcrt.yml b/.github/workflows/msvcrt.yml index 187bc312..663a9d48 100644 --- a/.github/workflows/msvcrt.yml +++ b/.github/workflows/msvcrt.yml @@ -76,7 +76,7 @@ jobs: DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-$(uname -m) NAME=llvm-mingw-$TAG-msvcrt-$DISTRO mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME - uses: actions/upload-artifact@v4 with: name: linux-msvcrt-x86_64-toolchain diff --git a/release-macos.sh b/release-macos.sh index 14e5a337..d4c82b16 100755 --- a/release-macos.sh +++ b/release-macos.sh @@ -33,7 +33,7 @@ rm -rf $DEST time CLEAN=1 SYNC=1 MACOS_REDIST=1 ./build-all.sh $DEST dir=$(pwd) cd $HOME -tar -Jcvf $dir/$RELNAME.tar.xz $RELNAME +gtar -Jcvf $dir/$RELNAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $RELNAME rm -rf $RELNAME cd $dir ls -lh $RELNAME.tar.xz diff --git a/release.sh b/release.sh index 40bdc72f..d621e7b3 100755 --- a/release.sh +++ b/release.sh @@ -30,7 +30,7 @@ fi time docker build -f Dockerfile . -t mstorsjo/llvm-mingw:latest -t mstorsjo/llvm-mingw:$TAG DISTRO=ubuntu-20.04-$(uname -m) -docker run --rm mstorsjo/llvm-mingw:latest sh -c "cd /opt && mv llvm-mingw llvm-mingw-$TAG-ucrt-$DISTRO && tar -Jcvf - llvm-mingw-$TAG-ucrt-$DISTRO" > llvm-mingw-$TAG-ucrt-$DISTRO.tar.xz +docker run --rm mstorsjo/llvm-mingw:latest sh -c "cd /opt && mv llvm-mingw llvm-mingw-$TAG-ucrt-$DISTRO && tar -Jcvf - --format=ustar --numeric-owner --owner=0 --group=0 llvm-mingw-$TAG-ucrt-$DISTRO" > llvm-mingw-$TAG-ucrt-$DISTRO.tar.xz if [ -n "$NATIVEONLY" ]; then exit 0 @@ -57,7 +57,7 @@ msvcrt_image=llvm-mingw-msvcrt-$(uuidgen) temp_images="$temp_images $msvcrt_image" time docker build -f Dockerfile.dev -t $msvcrt_image --build-arg DEFAULT_CRT=msvcrt . -docker run --rm $msvcrt_image sh -c "cd /opt && mv llvm-mingw llvm-mingw-$TAG-msvcrt-$DISTRO && tar -Jcvf - llvm-mingw-$TAG-msvcrt-$DISTRO" > llvm-mingw-$TAG-msvcrt-$DISTRO.tar.xz +docker run --rm $msvcrt_image sh -c "cd /opt && mv llvm-mingw llvm-mingw-$TAG-msvcrt-$DISTRO && tar -Jcvf - --format=ustar --numeric-owner --owner=0 --group=0 llvm-mingw-$TAG-msvcrt-$DISTRO" > llvm-mingw-$TAG-msvcrt-$DISTRO.tar.xz for arch in i686 x86_64; do temp=$(uuidgen) From 4716da81257ea6d032d6292298fdc1f9a8ebb5a7 Mon Sep 17 00:00:00 2001 From: Norbert Lange Date: Tue, 16 Jan 2024 21:17:32 +0100 Subject: [PATCH 2/3] attempt to use reproducible timestamps This sets SOURCE_DATE_EPOCH to the commit timestamp, toolchains should pick up timestamps from this variable. The same sources ideally would produce the same binary, see https://reproducible-builds.org. Set the timestamps within the tar archives to the start of the CI pipeline. Sort the archive contents. --- .github/workflows/build.yml | 31 ++++++++++++++++++++++++++----- .github/workflows/msvcrt.yml | 7 ++++++- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b437564c..01046053 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,13 @@ jobs: MINGW_W64_VERSION: ${{steps.get-versions.outputs.MINGW_W64_VERSION}} PYTHON_VERSION_MINGW: ${{steps.get-versions.outputs.PYTHON_VERSION_MINGW}} TAG: ${{steps.get-tag.outputs.TAG}} + COMMIT_DATE_UNIX: ${{steps.get-tag.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{steps.get-tag.outputs.BUILD_DATE}} + BUILD_DATE_UNIX: ${{steps.get-tag.outputs.BUILD_DATE_UNIX}} steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: . - name: Select build tag id: get-tag run: | @@ -36,6 +42,9 @@ jobs: TAG=$(TZ=UTC date +%Y%m%d) fi echo TAG=$TAG >> $GITHUB_OUTPUT + echo COMMIT_DATE_UNIX=$(git log -1 --pretty=%ct $GITHUB_SHA) >> $GITHUB_OUTPUT + echo BUILD_DATE=$(date -u '+%FT%TZ') >> $GITHUB_OUTPUT + echo BUILD_DATE_UNIX=$(date -d "${BUILD_DATE}" +%s) >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT cat $GITHUB_OUTPUT >> parameters.txt - name: Check latest version @@ -66,6 +75,8 @@ jobs: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}} TAG: ${{needs.prepare.outputs.TAG}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} run: | sudo apt-get update && sudo apt-get install ninja-build # Skip dynamic library dependencies that might make it harder to @@ -77,7 +88,7 @@ jobs: DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-$(uname -m) NAME=llvm-mingw-$TAG-ucrt-$DISTRO mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME - uses: actions/upload-artifact@v4 with: name: linux-ucrt-x86_64-toolchain @@ -97,6 +108,7 @@ jobs: env: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} run: | sudo apt-get update && sudo apt-get install ninja-build g++-aarch64-linux-gnu ./build-all.sh $(pwd)/install/llvm-mingw --disable-clang-tools-extra --no-runtimes --host=aarch64-linux-gnu @@ -119,12 +131,13 @@ jobs: - name: Package the toolchain env: TAG: ${{needs.prepare.outputs.TAG}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} run: | cd install DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-aarch64 NAME=llvm-mingw-$TAG-ucrt-$DISTRO mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME - uses: actions/upload-artifact@v4 with: name: linux-ucrt-aarch64-toolchain @@ -149,6 +162,8 @@ jobs: env: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} run: | sudo apt-get update && sudo apt-get install ninja-build # Skip dynamic library dependencies that might make it harder to @@ -157,7 +172,7 @@ jobs: LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLVM_ENABLE_TERMINFO=OFF" ./build-all.sh $(pwd)/install/llvm-mingw --disable-clang-tools-extra --disable-lldb --enable-asserts .github/workflows/store-version.sh install/llvm-mingw/versions.txt cd install - tar -Jcf ../llvm-mingw-linux.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 llvm-mingw + tar -Jcf ../llvm-mingw-linux.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" llvm-mingw - uses: actions/upload-artifact@v4 with: name: linux-asserts-toolchain @@ -177,6 +192,8 @@ jobs: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}} TAG: ${{needs.prepare.outputs.TAG}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} run: | brew install ninja gnu-tar # Disable zstd and python. Both are available on the runners, but @@ -189,7 +206,7 @@ jobs: cd install NAME=llvm-mingw-$TAG-ucrt-macos-universal mv llvm-mingw $NAME - gtar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME + gtar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME - uses: actions/upload-artifact@v4 with: name: macos-ucrt-toolchain @@ -233,6 +250,8 @@ jobs: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}} TAG: ${{needs.prepare.outputs.TAG}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} run: | ./build-all.sh $(pwd)/install/llvm-mingw --disable-clang-tools-extra --disable-lldb .github/workflows/store-version.sh install/llvm-mingw/versions.txt @@ -241,7 +260,7 @@ jobs: cd install NAME=llvm-mingw-$TAG-ucrt-msys2-${{matrix.sys}} mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME - uses: actions/upload-artifact@v4 with: name: msys2-${{matrix.sys}}-toolchain @@ -281,6 +300,8 @@ jobs: MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}} PYTHON_VERSION_MINGW: ${{needs.prepare.outputs.PYTHON_VERSION_MINGW}} TAG: ${{needs.prepare.outputs.TAG}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} run: | sudo apt-get update && sudo apt-get install autoconf-archive ninja-build ./build-cross-tools.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw ${{matrix.arch}} --with-python diff --git a/.github/workflows/msvcrt.yml b/.github/workflows/msvcrt.yml index 663a9d48..fd62ca10 100644 --- a/.github/workflows/msvcrt.yml +++ b/.github/workflows/msvcrt.yml @@ -24,6 +24,9 @@ jobs: MINGW_W64_VERSION: ${{steps.get-parameters.outputs.MINGW_W64_VERSION}} PYTHON_VERSION_MINGW: ${{steps.get-parameters.outputs.PYTHON_VERSION_MINGW}} TAG: ${{steps.get-parameters.outputs.TAG}} + COMMIT_DATE_UNIX: ${{steps.get-parameters.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{steps.get-parameters.outputs.BUILD_DATE}} + BUILD_DATE_UNIX: ${{steps.get-parameters.outputs.BUILD_DATE_UNIX}} steps: - name: Download build parameters uses: dawidd6/action-download-artifact@v3 @@ -68,6 +71,8 @@ jobs: LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}} MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}} TAG: ${{needs.prepare.outputs.TAG}} + SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}} + BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}} run: | sudo apt-get update && sudo apt-get install ninja-build ./build-all.sh $(pwd)/install/llvm-mingw --no-tools --wipe-runtimes --with-default-msvcrt=msvcrt @@ -76,7 +81,7 @@ jobs: DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-$(uname -m) NAME=llvm-mingw-$TAG-msvcrt-$DISTRO mv llvm-mingw $NAME - tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $NAME + tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME - uses: actions/upload-artifact@v4 with: name: linux-msvcrt-x86_64-toolchain From f4aadbfd5c14f60a797485848ec9639b990d99f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Tue, 13 Feb 2024 00:19:55 +0200 Subject: [PATCH 3/3] release-macos: Don't assume that gtar is available on macOS --- release-macos.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release-macos.sh b/release-macos.sh index d4c82b16..503f3c5c 100755 --- a/release-macos.sh +++ b/release-macos.sh @@ -33,7 +33,12 @@ rm -rf $DEST time CLEAN=1 SYNC=1 MACOS_REDIST=1 ./build-all.sh $DEST dir=$(pwd) cd $HOME -gtar -Jcvf $dir/$RELNAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 $RELNAME +TAR=tar +if command -v gtar >/dev/null; then + TAR_FLAGS="--numeric-owner --owner=0 --group=0" + TAR=gtar +fi +$TAR -Jcvf $dir/$RELNAME.tar.xz --format=ustar $TAR_FLAGS $RELNAME rm -rf $RELNAME cd $dir ls -lh $RELNAME.tar.xz