Smaller CI intermediate tarballs (option 2)

.github/workflows/bwrap.yml

@@ -42,21 +42,23 @@ jobs:
           path: |
             distfiles
           key: cache-${{ hashFiles('steps/*/sources') }}
-      - name: Run bootstrap
-        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
-      - name: Archive created packages
-        if: failure() # archive failed builds progress
+      - name: Run seed bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci start
+      - name: Tar seed image
+        run: tar -cf seed_image.tar --exclude "target/external/distfiles" target
+      - name: Archive seed_image
         uses: actions/upload-artifact@v3
         with:
-          name: packages
-          path: target/external/repo/**
-      - name: Tar pass1 image
-        run: tar -cf pass1_image.tar target
-      - name: Archive pass1_image
+          name: seed_image
+          path: seed_image.tar
+      - name: Run pass1 bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue
+      - name: Archive created packages
+        if: always() # archive both failed and successful builds
         uses: actions/upload-artifact@v3
         with:
-          name: internal_pass1_image
-          path: pass1_image.tar
+          name: packages_pass1
+          path: target/external/repo/**
 
   pass2:
     name: Run up to Python bootstrap under bubblewrap
@@ -72,12 +74,17 @@ jobs:
           # There is a strange bug(?) in nongnu, when you clone a git repository
           # against a commit != HEAD with depth=1, it errors out.
           fetch-depth: 0
-      - name: Get pass1_image
+      - name: Get seed_image
         uses: actions/download-artifact@v3
         with:
-          name: internal_pass1_image
-      - name: Extract pass1_image
-        run: tar -xf pass1_image.tar
+          name: seed_image
+      - name: Extract seed_image
+        run: tar -xf seed_image.tar
+      - name: Get pass1 packages
+        uses: actions/download-artifact@v3
+        with:
+          name: packages_pass1
+          path: target/external/repo-preseeded/
       - name: Query cache for sources
         id: cache
         uses: actions/cache/restore@v3
@@ -95,21 +102,16 @@ jobs:
           path: |
             distfiles
           key: cache-${{ hashFiles('steps/*/sources') }}
-      - name: Run bootstrap
-        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
+      - name: Preseed bootstrap with pass1
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue
+      - name: Run pass2 bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue
       - name: Archive created packages
-        if: failure() # archive failed builds progress
+        if: always() # archive both failed and successful builds
         uses: actions/upload-artifact@v3
         with:
-          name: internal_packages_pass2
+          name: packages_pass2
           path: target/external/repo/**
-      - name: Tar pass2 image
-        run: tar -cf pass2_image.tar target
-      - name: Archive pass2_image
-        uses: actions/upload-artifact@v3
-        with:
-          name: internal_pass2_image
-          path: pass2_image.tar
 
   pass3:
     name: Run remaining builds under bubblewrap
@@ -125,12 +127,17 @@ jobs:
           # There is a strange bug(?) in nongnu, when you clone a git repository
           # against a commit != HEAD with depth=1, it errors out.
           fetch-depth: 0
-      - name: Get pass2_image
+      - name: Get seed_image
+        uses: actions/download-artifact@v3
+        with:
+          name: seed_image
+      - name: Extract seed_image
+        run: tar -xf seed_image.tar
+      - name: Get pass2 packages
         uses: actions/download-artifact@v3
         with:
-          name: internal_pass2_image
-      - name: Extract pass2_image
-        run: tar -xf pass2_image.tar
+          name: packages_pass2
+          path: target/external/repo-preseeded/
       - name: Query cache for sources
         id: cache
         uses: actions/cache/restore@v3
@@ -148,8 +155,12 @@ jobs:
           path: |
             distfiles
           key: cache-${{ hashFiles('steps/*/sources') }}
-      - name: Run bootstrap
-        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
+      - name: Preseed bootstrap with pass1
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue
+      - name: Preseed bootstrap with pass2
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue
+      - name: Run pass3 bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci continue
       - name: Archive created packages
         if: always() # archive both failed and successful builds
         uses: actions/upload-artifact@v3

rootfs.py

@@ -185,7 +185,7 @@ def bootstrap(args, generator, target, size):
 
     elif args.bwrap:
         init = '/init'
-        if not args.internal_ci or args.internal_ci == "pass1":
+        if not args.internal_ci or args.internal_ci == "start":
             generator.prepare(target, using_kernel=False)
 
             arch = stage0_arch_map.get(args.arch, args.arch)

steps/SHA256SUMS.pkgs

@@ -68,7 +68,7 @@ f650c3b24de6edd49cd662c9e9ce11f7b0b5ea6df66d561b46a032b08fc34faa  kbd-1.15_0.tar
 36550df491767bb24d2ccab304ce70a3b4956e7c0c0e0c343d922fd57cdafbdd  libatomic_ops-7.6.10_0.tar.bz2
 b39826742e236890f3562cdf19492e7ef4224b271f3e75ddeab1f07982b03ebe  libffi-3.3_0.tar.bz2
 daae709e98d2df2190d1d13b4e86f7f3fe90fa7a975282fe0bb03289b6539f29  libtool-2.2.4_0.tar.bz2
-6cefa575362149620f8008a32c8af54f0198a18bc6ab910bd3cead196c1507d7  libtool-2.4.7_0.tar.bz2
+fb534c4c93bdcbe0ad67b19e202bf5fb7efc34adf73c2f81968b814fc99f7426  libtool-2.4.7_0.tar.bz2
 503007bbcddcf4e49d26514c59b4c9501f8b42f0c994a59dfdc388b1ae6b7900  libunistring-0.9.10_0.tar.bz2
 fd2264a08cbeeb28f3e2580cba174376a154c77b0ef6cbf67b55fd13c0987f52  linux-4.9.10_0.tar.bz2
 d15c922973c15a8206e09020e8cfe6a78f7e93614de212f2b37ff80163799c6c  linux-headers-4.9.10_0.tar.bz2
@@ -90,7 +90,7 @@ c490016e49bbf77e7f63071f7aa60e8290a0c67f017846def1c3f65bd10d5712  openssl-1.1.1l
 b86fc586d72a6d204d0b9017afe17aa2439529e13f43befc56648b78de17c8f8  perl5.004-05_0.tar.bz2
 451ae6bb6e996a1c857fe92ec379654413fadec180c6477cf5fd4b5d7a079e26  perl5.005-03_0.tar.bz2
 74d64a8af080022432fa94dba449090419d25b103d247710dc0b6102a4ad86a6  perl-5.10.1_0.tar.bz2
-fdccd3ba27a44d2149f159040414a04b39bfc72673ba36f50051b61199cc425c  perl-5.32.1_0.tar.bz2
+71ad3cadba5801cb19d4520825d4b3606713807b1eaa5eb3c49b3149bc2675ad  perl-5.32.1_0.tar.bz2
 9fa8850897248d77d554caa2b3d52e7890308cbf28750ffc24c09f498aee563c  perl-5.6.2_0.tar.bz2
 1b9d4260edf7b2241d10e4c4ad17d0f90047bd4bf42f2487a7133902529e9dfe  pkg-config-0.29.2_0.tar.bz2
 1e882c3206f9d1de2a9be8b5c6ae4cc65e80a4de607bd521058577bf4169c0e9  python-2.0.1_0.tar.bz2

steps/libtool-2.4.7/pass1.sh

@@ -35,7 +35,7 @@ src_configure() {
 }
 
 src_compile() {
-    make "${MAKEJOBS}" AUTOM4TE=autom4te-2.69 MAKEINFO=true
+    make "${MAKEJOBS}" AUTOM4TE=autom4te-2.69 AUTOMAKE=automake-1.15 ACLOCAL=aclocal-1.15 AUTOCONF=autoconf-2.69 AUTOHEADER=autoheader-2.69 MAKEINFO=true
 }
 
 src_install() {

steps/manifest

@@ -50,6 +50,7 @@ build: heirloom-devtools-070527
 build: bash-2.05b
 improve: setup_repo
 improve: update_env
+jump: break ( INTERNAL_CI == start )
 build: flex-2.5.11
 build: tcc-0.9.27
 improve: musl_libdir
@@ -115,7 +116,7 @@ define: BUILD_LINUX = ( CHROOT == False || BUILD_KERNELS == True )
 build: kexec-linux-1.0.0 ( BUILD_LINUX == True )
 build: kexec-tools-2.0.22 ( BUILD_LINUX == True )
 build: linux-4.9.10 ( BUILD_LINUX == True )
-jump: break ( INTERNAL_CI == pass1 )
+jump: break ( INTERNAL_CI == start )
 jump: linux ( CHROOT == False )
 improve: finalize_fhs
 build: musl-1.2.4
@@ -175,7 +176,7 @@ build: python-3.3.7
 build: python-3.4.10
 build: python-3.8.16
 build: python-3.11.1
-jump: break ( INTERNAL_CI == pass1 ) # scripts are generated in pass1
+jump: break ( INTERNAL_CI == start )
 build: gcc-10.4.0
 build: binutils-2.41
 build: gcc-13.1.0

steps/musl-1.1.24/pass3.sh

@@ -28,6 +28,5 @@ src_compile() {
 }
 
 src_install() {
-    rm -rf "${PREFIX}/include"
     make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" install
 }