From 643577e049149e0d618f531cf242216043d212f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= Date: Wed, 27 Dec 2023 19:14:58 +0100 Subject: [PATCH 1/3] Revert to seed tarball + packages for CI pass transitions The seed tarball is created as early as possible, to minimize its size. --- .github/workflows/bwrap.yml | 75 +++++++++++++++++++++---------------- rootfs.py | 2 +- steps/manifest | 5 ++- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 66cea846..302455da 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -42,21 +42,23 @@ jobs: path: | distfiles key: cache-${{ hashFiles('steps/*/sources') }} - - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 - - name: Archive created packages - if: failure() # archive failed builds progress + - name: Run seed bootstrap + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci start + - name: Tar seed image + run: tar -cf seed_image.tar target + - name: Archive seed_image uses: actions/upload-artifact@v3 with: - name: packages - path: target/external/repo/** - - name: Tar pass1 image - run: tar -cf pass1_image.tar target - - name: Archive pass1_image + name: seed_image + path: seed_image.tar + - name: Run pass1 bootstrap + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue + - name: Archive created packages + if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 with: - name: internal_pass1_image - path: pass1_image.tar + name: packages_pass1 + path: target/external/repo/** pass2: name: Run up to Python bootstrap under bubblewrap @@ -72,12 +74,17 @@ jobs: # There is a strange bug(?) in nongnu, when you clone a git repository # against a commit != HEAD with depth=1, it errors out. fetch-depth: 0 - - name: Get pass1_image + - name: Get seed_image uses: actions/download-artifact@v3 with: - name: internal_pass1_image - - name: Extract pass1_image - run: tar -xf pass1_image.tar + name: seed_image + - name: Extract seed_image + run: tar -xf seed_image.tar + - name: Get pass1 packages + uses: actions/download-artifact@v3 + with: + name: packages_pass1 + path: target/external/repo-preseeded/ - name: Query cache for sources id: cache uses: actions/cache/restore@v3 @@ -95,21 +102,16 @@ jobs: path: | distfiles key: cache-${{ hashFiles('steps/*/sources') }} - - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 + - name: Preseed bootstrap with pass1 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue + - name: Run pass2 bootstrap + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue - name: Archive created packages - if: failure() # archive failed builds progress + if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 with: - name: internal_packages_pass2 + name: packages_pass2 path: target/external/repo/** - - name: Tar pass2 image - run: tar -cf pass2_image.tar target - - name: Archive pass2_image - uses: actions/upload-artifact@v3 - with: - name: internal_pass2_image - path: pass2_image.tar pass3: name: Run remaining builds under bubblewrap @@ -125,12 +127,17 @@ jobs: # There is a strange bug(?) in nongnu, when you clone a git repository # against a commit != HEAD with depth=1, it errors out. fetch-depth: 0 - - name: Get pass2_image + - name: Get seed_image + uses: actions/download-artifact@v3 + with: + name: seed_image + - name: Extract seed_image + run: tar -xf seed_image.tar + - name: Get pass2 packages uses: actions/download-artifact@v3 with: - name: internal_pass2_image - - name: Extract pass2_image - run: tar -xf pass2_image.tar + name: packages_pass2 + path: target/external/repo-preseeded/ - name: Query cache for sources id: cache uses: actions/cache/restore@v3 @@ -148,8 +155,12 @@ jobs: path: | distfiles key: cache-${{ hashFiles('steps/*/sources') }} - - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 + - name: Preseed bootstrap with pass1 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue + - name: Preseed bootstrap with pass2 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue + - name: Run pass3 bootstrap + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue - name: Archive created packages if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 diff --git a/rootfs.py b/rootfs.py index a08d27dc..91442de4 100755 --- a/rootfs.py +++ b/rootfs.py @@ -185,7 +185,7 @@ def bootstrap(args, generator, target, size): elif args.bwrap: init = '/init' - if not args.internal_ci or args.internal_ci == "pass1": + if not args.internal_ci or args.internal_ci == "start": generator.prepare(target, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) diff --git a/steps/manifest b/steps/manifest index b5aed41a..005ee87f 100644 --- a/steps/manifest +++ b/steps/manifest @@ -50,6 +50,7 @@ build: heirloom-devtools-070527 build: bash-2.05b improve: setup_repo improve: update_env +jump: break ( INTERNAL_CI == start ) build: flex-2.5.11 build: tcc-0.9.27 improve: musl_libdir @@ -115,7 +116,7 @@ define: BUILD_LINUX = ( CHROOT == False || BUILD_KERNELS == True ) build: kexec-linux-1.0.0 ( BUILD_LINUX == True ) build: kexec-tools-2.0.22 ( BUILD_LINUX == True ) build: linux-4.9.10 ( BUILD_LINUX == True ) -jump: break ( INTERNAL_CI == pass1 ) +jump: break ( INTERNAL_CI == start ) jump: linux ( CHROOT == False ) improve: finalize_fhs build: musl-1.2.4 @@ -175,7 +176,7 @@ build: python-3.3.7 build: python-3.4.10 build: python-3.8.16 build: python-3.11.1 -jump: break ( INTERNAL_CI == pass1 ) # scripts are generated in pass1 +jump: break ( INTERNAL_CI == start ) build: gcc-10.4.0 build: binutils-2.41 build: gcc-13.1.0 From 33db7bad1869ff42eec54b26cb9466e760cc9002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= Date: Wed, 27 Dec 2023 19:18:40 +0100 Subject: [PATCH 2/3] Exclude distfiles from seed image --- .github/workflows/bwrap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 302455da..94644d58 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -45,7 +45,7 @@ jobs: - name: Run seed bootstrap run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci start - name: Tar seed image - run: tar -cf seed_image.tar target + run: tar -cf seed_image.tar --exclude "target/external/distfiles" target - name: Archive seed_image uses: actions/upload-artifact@v3 with: From 5f60a8f50b9dbf63a9f4047936f76f2989c74a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= Date: Thu, 28 Dec 2023 06:56:47 +0100 Subject: [PATCH 3/3] Fix checksum inconsistencies between staged and unstaged bwrap bootstrap - libtool needs autoconf (& possibly other autotools) at make time, to generate correct & reproducible manpages - musl-1.1.24 shouldn't nuke /usr/include in its src_install(), as it will remove headers used by other packages later, notably perl. (Also, touching the live filesystem in src_install() will be skipped if the package is preseeded instead, causing inconsistency.) --- steps/SHA256SUMS.pkgs | 4 ++-- steps/libtool-2.4.7/pass1.sh | 2 +- steps/musl-1.1.24/pass3.sh | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/steps/SHA256SUMS.pkgs b/steps/SHA256SUMS.pkgs index 8e9124db..688df2e0 100644 --- a/steps/SHA256SUMS.pkgs +++ b/steps/SHA256SUMS.pkgs @@ -68,7 +68,7 @@ f650c3b24de6edd49cd662c9e9ce11f7b0b5ea6df66d561b46a032b08fc34faa kbd-1.15_0.tar 36550df491767bb24d2ccab304ce70a3b4956e7c0c0e0c343d922fd57cdafbdd libatomic_ops-7.6.10_0.tar.bz2 b39826742e236890f3562cdf19492e7ef4224b271f3e75ddeab1f07982b03ebe libffi-3.3_0.tar.bz2 daae709e98d2df2190d1d13b4e86f7f3fe90fa7a975282fe0bb03289b6539f29 libtool-2.2.4_0.tar.bz2 -6cefa575362149620f8008a32c8af54f0198a18bc6ab910bd3cead196c1507d7 libtool-2.4.7_0.tar.bz2 +fb534c4c93bdcbe0ad67b19e202bf5fb7efc34adf73c2f81968b814fc99f7426 libtool-2.4.7_0.tar.bz2 503007bbcddcf4e49d26514c59b4c9501f8b42f0c994a59dfdc388b1ae6b7900 libunistring-0.9.10_0.tar.bz2 fd2264a08cbeeb28f3e2580cba174376a154c77b0ef6cbf67b55fd13c0987f52 linux-4.9.10_0.tar.bz2 d15c922973c15a8206e09020e8cfe6a78f7e93614de212f2b37ff80163799c6c linux-headers-4.9.10_0.tar.bz2 @@ -90,7 +90,7 @@ c490016e49bbf77e7f63071f7aa60e8290a0c67f017846def1c3f65bd10d5712 openssl-1.1.1l b86fc586d72a6d204d0b9017afe17aa2439529e13f43befc56648b78de17c8f8 perl5.004-05_0.tar.bz2 451ae6bb6e996a1c857fe92ec379654413fadec180c6477cf5fd4b5d7a079e26 perl5.005-03_0.tar.bz2 74d64a8af080022432fa94dba449090419d25b103d247710dc0b6102a4ad86a6 perl-5.10.1_0.tar.bz2 -fdccd3ba27a44d2149f159040414a04b39bfc72673ba36f50051b61199cc425c perl-5.32.1_0.tar.bz2 +71ad3cadba5801cb19d4520825d4b3606713807b1eaa5eb3c49b3149bc2675ad perl-5.32.1_0.tar.bz2 9fa8850897248d77d554caa2b3d52e7890308cbf28750ffc24c09f498aee563c perl-5.6.2_0.tar.bz2 1b9d4260edf7b2241d10e4c4ad17d0f90047bd4bf42f2487a7133902529e9dfe pkg-config-0.29.2_0.tar.bz2 1e882c3206f9d1de2a9be8b5c6ae4cc65e80a4de607bd521058577bf4169c0e9 python-2.0.1_0.tar.bz2 diff --git a/steps/libtool-2.4.7/pass1.sh b/steps/libtool-2.4.7/pass1.sh index 9e34141d..4a27412e 100755 --- a/steps/libtool-2.4.7/pass1.sh +++ b/steps/libtool-2.4.7/pass1.sh @@ -35,7 +35,7 @@ src_configure() { } src_compile() { - make "${MAKEJOBS}" AUTOM4TE=autom4te-2.69 MAKEINFO=true + make "${MAKEJOBS}" AUTOM4TE=autom4te-2.69 AUTOMAKE=automake-1.15 ACLOCAL=aclocal-1.15 AUTOCONF=autoconf-2.69 AUTOHEADER=autoheader-2.69 MAKEINFO=true } src_install() { diff --git a/steps/musl-1.1.24/pass3.sh b/steps/musl-1.1.24/pass3.sh index 6a2342db..6f7c1ae2 100755 --- a/steps/musl-1.1.24/pass3.sh +++ b/steps/musl-1.1.24/pass3.sh @@ -28,6 +28,5 @@ src_compile() { } src_install() { - rm -rf "${PREFIX}/include" make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" install }