From dc62d8242cafddb438e149fcd81e3272eea73da1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 20:50:47 +0100
Subject: [PATCH 01/13] Create external.img only when needed

---
 lib/generator.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index e1dbac55..3d33650c 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -57,8 +57,6 @@ def prepare(self, using_kernel=False, kernel_bootstrap=False):
             if self.repo_path or self.external_sources:
                 self.tmpdir.add_disk("external", filesystem="ext3")
                 self.tmpdir.mount_disk("external", "external")
-            else:
-                self.tmpdir.add_disk("external", tabletype="none")
         elif using_kernel:
             self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
             self.tmpdir.add_disk("disk", filesystem="ext3")

From b4d9c5e7cb4bb3e1bee9e6105c90c9c0e3b0eb35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 21:59:00 +0100
Subject: [PATCH 02/13] Disk creation improvements

* Support specifying the size of the target disk image for qemu
* For bare metal, only pad the image to the next megabyte
* Use truncate() to extend images, instead of writing zeros (faster)
* Return None from get_disk() with nonexistent name
* Leave 1MiB on non-boot disks, or 1GiB on boot disks, unpartitioned
  (for proper 4K alignment and to help preserve the srcfs or boot
  partition creation)
* Fix qemu invocation when an external.img is not used
* Make -qr work with kernel bootstrap (will need kexec-fiwix fix)
---
 lib/generator.py | 24 ++++++++++++++----------
 lib/tmpdir.py    | 17 ++++++++++++++---
 lib/utils.py     |  5 +++--
 rootfs.py        | 42 ++++++++++++++++++++++++++++++------------
 4 files changed, 61 insertions(+), 27 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index 3d33650c..a5cadc1a 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -33,7 +33,7 @@ def __init__(self, tmpdir, arch, external_sources,
         self.tmp_dir = tmpdir.path
         self.external_dir = os.path.join(self.tmp_dir, 'external')
 
-    def prepare(self, using_kernel=False, kernel_bootstrap=False):
+    def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
         """
         Prepare basic media of live-bootstrap.
         /steps -- contains steps to be built
@@ -57,9 +57,14 @@ def prepare(self, using_kernel=False, kernel_bootstrap=False):
             if self.repo_path or self.external_sources:
                 self.tmpdir.add_disk("external", filesystem="ext3")
                 self.tmpdir.mount_disk("external", "external")
+            else:
+                self.external_dir = os.path.join(self.tmp_dir, 'external')
         elif using_kernel:
             self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
-            self.tmpdir.add_disk("disk", filesystem="ext3")
+            self.tmpdir.add_disk("disk",
+                            filesystem="ext3",
+                            size=(target_size + "M") if target_size else "16G",
+                            bootable=True)
             self.tmpdir.mount_disk("disk", "disk")
             self.external_dir = os.path.join(self.tmp_dir, 'external')
 
@@ -86,7 +91,7 @@ def prepare(self, using_kernel=False, kernel_bootstrap=False):
             shutil.copytree(self.repo_path, repo_dir)
 
         if kernel_bootstrap:
-            self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'))
+            self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'), target_size)
 
         if kernel_bootstrap and (self.external_sources or self.repo_path):
             self.tmpdir.umount_disk('external')
@@ -149,7 +154,7 @@ def create_fiwix_file_list(self):
     def distfiles(self):
         """Copy in distfiles"""
         def copy_no_network_distfiles(out):
-            # Note that no network == no disk for kernel bootstrap mode
+            # Note that "no disk" implies "no network" for kernel bootstrap mode
             pre_src_path = os.path.join(self.git_dir, 'steps', 'pre-network-sources')
             with open(pre_src_path, 'r', encoding="utf-8") as source_list:
                 for file in source_list.readlines():
@@ -222,7 +227,7 @@ def append_srcfs(self, image_file):
 
         os.chdir(save_cwd)
 
-    def create_builder_hex0_disk_image(self, image_file_name):
+    def create_builder_hex0_disk_image(self, image_file_name, size):
         """Create builder-hex0 disk image"""
         shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds',
                                      'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
@@ -250,11 +255,10 @@ def create_builder_hex0_disk_image(self, image_file_name):
             image_file.write(b'\0' * round_up)
         current_size += round_up
 
-        # fill file with zeros up to desired size, one megabyte at a time
-        with open(image_file_name, 'ab') as image_file:
-            while current_size < 16384 * megabyte:
-                image_file.write(b'\0' * megabyte)
-                current_size += megabyte
+        # extend file up to desired size
+        if current_size < size * megabyte:
+            with open(image_file_name, 'ab') as image_file:
+                image_file.truncate(size * megabyte)
 
     def check_file(self, file_name, expected_hash):
         """Check hash of downloaded source file."""
diff --git a/lib/tmpdir.py b/lib/tmpdir.py
index f72286f7..954235cc 100644
--- a/lib/tmpdir.py
+++ b/lib/tmpdir.py
@@ -60,10 +60,21 @@ def tmpfs(self, size="8G"):
         self._type = TmpType.TMPFS
 
     # pylint: disable=too-many-arguments
-    def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=None):
+    def add_disk(self,
+                 name,
+                 size="16G",
+                 filesystem="ext4",
+                 tabletype="msdos",
+                 bootable=False,
+                 mkfs_args=None):
         """Add a disk"""
         disk_path = os.path.join(self.path, f"{name}.img")
-        self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args)
+        self._disks[name] = create_disk(disk_path,
+                                        tabletype,
+                                        filesystem,
+                                        size,
+                                        bootable,
+                                        mkfs_args)
         self._disk_filesystems[name] = filesystem
         # Allow executing user to access it
         run_as_root("chown", getpass.getuser(), self._disks[name])
@@ -87,4 +98,4 @@ def umount_disk(self, name):
 
     def get_disk(self, name):
         """Get the path to a device of a disk"""
-        return self._disks[name]
+        return self._disks.get(name)
diff --git a/lib/utils.py b/lib/utils.py
index a80d656c..d88ae4d4 100755
--- a/lib/utils.py
+++ b/lib/utils.py
@@ -31,7 +31,8 @@ def run_as_root(*args, **kwargs):
         return run("sudo", *args, **kwargs)
     return run(*args, **kwargs)
 
-def create_disk(image, disk_type, fs_type, size, mkfs_args=None):
+# pylint: disable=too-many-arguments
+def create_disk(image, disk_type, fs_type, size, bootable=False, mkfs_args=None):
     """Create a disk image, with a filesystem on it"""
     if mkfs_args is None:
         mkfs_args = []
@@ -42,7 +43,7 @@ def create_disk(image, disk_type, fs_type, size, mkfs_args=None):
     # Create the partition
     if disk_type != "none":
         run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart',
-                'primary', fs_type, '0%', '100%')
+                'primary', fs_type, '1GiB' if bootable else '1MiB', '100%')
         run_as_root('partprobe', loop_dev)
         run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args)
     return loop_dev
diff --git a/rootfs.py b/rootfs.py
index 648b1d08..d278294a 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -39,7 +39,7 @@ def create_configuration_file(args):
             if args.repo or args.external_sources:
                 config.write("DISK=sdb1\n")
             else:
-                config.write("DISK=sdb\n")
+                config.write("DISK=sda\n")
             config.write("KERNEL_BOOTSTRAP=True\n")
         else:
             config.write("DISK=sda1\n")
@@ -98,6 +98,8 @@ def main():
                         default="qemu-system-x86_64")
     parser.add_argument("-qr", "--qemu-ram", help="Memory (in megabytes) allocated to QEMU VM",
                         default=4096)
+    parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)",
+                        default="16G")
     parser.add_argument("-qk", "--kernel", help="Custom sysa kernel to use")
 
     parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
@@ -136,6 +138,15 @@ def check_types():
     if int(args.cores) < 1:
         raise ValueError("Must use one or more cores.")
 
+    # Target image size validation
+    if args.qemu:
+        if int(str(args.target_size).rstrip('gGmM')) < 1:
+            raise ValueError("Please specify a positive target size for qemu.")
+        args.target_size = (int(str(args.target_size).rstrip('gGmM')) *
+            (1024 if str(args.target_size).lower().endswith('g') else 1))
+    else:
+        args.target_size = 0
+
     # bootstrap.cfg
     try:
         os.remove(os.path.join('sysa', 'bootstrap.cfg'))
@@ -158,9 +169,9 @@ def check_types():
                           repo_path=args.repo,
                           early_preseed=args.early_preseed)
 
-    bootstrap(args, generator, tmpdir)
+    bootstrap(args, generator, tmpdir, args.target_size)
 
-def bootstrap(args, generator, tmpdir):
+def bootstrap(args, generator, tmpdir, size):
     """Kick off bootstrap process."""
     print(f"Bootstrapping {args.arch} -- SysA")
     if args.chroot:
@@ -224,18 +235,18 @@ def bootstrap(args, generator, tmpdir):
 
     elif args.bare_metal:
         if args.kernel:
-            generator.prepare(using_kernel=True)
+            generator.prepare(using_kernel=True, target_size=size)
             print("Please:")
             print("  1. Take tmp/initramfs and your kernel, boot using this.")
             print("  2. Take tmp/disk.img and put this on a writable storage medium.")
         else:
-            generator.prepare(kernel_bootstrap=True)
+            generator.prepare(kernel_bootstrap=True, target_size=size)
             print("Please:")
             print("  1. Take tmp/disk.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:
-            generator.prepare(using_kernel=True)
+            generator.prepare(using_kernel=True, target_size=size)
 
             run(args.qemu_cmd,
                 '-enable-kvm',
@@ -249,17 +260,24 @@ def bootstrap(args, generator, tmpdir):
                 '-nographic',
                 '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
         else:
-            generator.prepare(kernel_bootstrap=True)
-            run(args.qemu_cmd,
+            generator.prepare(kernel_bootstrap=True, target_size=size)
+            arg_list = [
                 '-enable-kvm',
-                '-m', "4G",
+                '-m', str(args.qemu_ram) + 'M',
                 '-smp', str(args.cores),
                 '-no-reboot',
-                '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw',
-                '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
+                '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw'
+            ]
+            if tmpdir.get_disk("external") is not None:
+                arg_list += [
+                    '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
+                ]
+            arg_list += [
                 '-machine', 'kernel-irqchip=split',
                 '-nic', 'user,ipv6=off,model=e1000',
-                '-nographic')
+                '-nographic'
+            ]
+            run(args.qemu_cmd, *arg_list)
 
 if __name__ == "__main__":
     main()

From c188185ad4525c5679ebc38f2819a1ecd629c202 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 22:09:04 +0100
Subject: [PATCH 03/13] Remove residual references to sysa/sysb/sysc from
 rootfs.py

---
 rootfs.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/rootfs.py b/rootfs.py
index d278294a..98a57949 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -30,7 +30,6 @@ def create_configuration_file(args):
     with open(config_path, "w", encoding="utf_8") as config:
         config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n")
         config.write(f"CHROOT={args.chroot or args.bwrap}\n")
-        config.write(f"CHROOT_ONLY_SYSA={args.bwrap}\n")
         config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n")
         config.write(f"JOBS={args.cores}\n")
         config.write(f"INTERNAL_CI={args.internal_ci}\n")
@@ -100,7 +99,7 @@ def main():
                         default=4096)
     parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)",
                         default="16G")
-    parser.add_argument("-qk", "--kernel", help="Custom sysa kernel to use")
+    parser.add_argument("-qk", "--kernel", help="Custom early kernel to use")
 
     parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
                         action="store_true")
@@ -149,13 +148,13 @@ def check_types():
 
     # bootstrap.cfg
     try:
-        os.remove(os.path.join('sysa', 'bootstrap.cfg'))
+        os.remove(os.path.join('steps', 'bootstrap.cfg'))
     except FileNotFoundError:
         pass
     if not args.no_create_config:
         create_configuration_file(args)
     else:
-        with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
+        with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
             pass
 
     # tmpdir
@@ -173,7 +172,7 @@ def check_types():
 
 def bootstrap(args, generator, tmpdir, size):
     """Kick off bootstrap process."""
-    print(f"Bootstrapping {args.arch} -- SysA")
+    print(f"Bootstrapping {args.arch}")
     if args.chroot:
         find_chroot = """
 import shutil
@@ -211,15 +210,16 @@ def bootstrap(args, generator, tmpdir, size):
                          init)
 
         if not args.internal_ci or args.internal_ci == "pass2" or args.internal_ci == "pass3":
-            shutil.copy2(os.path.join('sysa', 'bootstrap.cfg'),
-                         os.path.join('tmp', 'sysa', 'sysc_image', 'usr', 'src', 'bootstrap.cfg'))
+            os.makedirs(os.path.join(generator.tmp_dir, 'stage2', 'steps'), exist_ok=True)
+            shutil.copy2(os.path.join('steps', 'bootstrap.cfg'),
+                         os.path.join(generator.tmp_dir, 'stage2', 'steps', 'bootstrap.cfg'))
             run('bwrap', '--unshare-user',
                          '--uid', '0',
                          '--gid', '0',
                          '--unshare-net' if args.external_sources else None,
                          '--clearenv',
                          '--setenv', 'PATH', '/usr/bin',
-                         '--bind', generator.tmp_dir + "/sysc_image", '/',
+                         '--bind', os.path.join(generator.tmp_dir, "stage2"), '/',
                          '--dir', '/dev',
                          '--dev-bind', '/dev/null', '/dev/null',
                          '--dev-bind', '/dev/zero', '/dev/zero',

From faad907fba73a3d081b905d380e96b9fe3c1030b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 22:11:02 +0100
Subject: [PATCH 04/13] Fix source_manifest.py in light of the recent refactor

There's no sysa or sysc anymore.
---
 source_manifest.py | 20 ++------------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/source_manifest.py b/source_manifest.py
index 6fa4e35f..81a1ba96 100755
--- a/source_manifest.py
+++ b/source_manifest.py
@@ -9,27 +9,11 @@
 
 import argparse
 
-from sysa import SysA
-from sysc import SysC
+from lib.generator import Generator
 
 def main():
     """Generate a source manifest for a system"""
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument("-s", "--system",
-                        help="Generate source manifest for the specified systems",
-                        choices=["sysa", "sysc"],
-                        nargs="+",
-                        action="extend",
-                        required=True)
-
-    args = parser.parse_args()
-
-    if "sysa" in args.system:
-        print(SysA.get_source_manifest())
-
-    if "sysc" in args.system:
-        print(SysC.get_source_manifest())
+    print(Generator.get_source_manifest())
 
 if __name__ == "__main__":
     main()

From 8d193df8231b9ce4737614ba039008c40a73b6cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 22:22:13 +0100
Subject: [PATCH 05/13] Fix printed image paths for bare-metal bootstrap

---
 rootfs.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/rootfs.py b/rootfs.py
index 98a57949..172065b8 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -236,13 +236,15 @@ def bootstrap(args, generator, tmpdir, size):
     elif args.bare_metal:
         if args.kernel:
             generator.prepare(using_kernel=True, target_size=size)
+            image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
-            print("  1. Take tmp/initramfs and your kernel, boot using this.")
-            print("  2. Take tmp/disk.img and put this on a writable storage medium.")
+            print(f"  1. Take {image_path}/initramfs and your kernel, boot using this.")
+            print(f"  2. Take {image_path}/disk.img and put this on a writable storage medium.")
         else:
             generator.prepare(kernel_bootstrap=True, target_size=size)
+            image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
-            print("  1. Take tmp/disk.img and write it to a boot drive and then boot it.")
+            print(f"  1. Take {image_path}/disk.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:

From a68ae62f9e50781eb123f0fcc4542f75707244dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 22:58:30 +0100
Subject: [PATCH 06/13] Download distfiles only when needed, based on manifest

Unless --external-sources is given, only download distfiles that
need to be included in srcfs. The rest will be downloaded anyway
by the bootstrap system once it gets network access.

To accomplish this, instead of searching steps for sources files,
we now parse steps/manifest. As a side effect, source_manifest.py
now outputs source files in the order they appear in the manifest.
---
 lib/generator.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index a5cadc1a..54e38599 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -100,7 +100,7 @@ def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
 
     def steps(self):
         """Copy in steps."""
-        source_manifest = self.get_source_manifest()
+        source_manifest = self.get_source_manifest(not self.external_sources)
         self.get_packages(source_manifest)
 
         shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
@@ -307,7 +307,7 @@ def get_packages(self, source_manifest):
             self.check_file(path, line[0])
 
     @classmethod
-    def get_source_manifest(cls):
+    def get_source_manifest(cls, pre_network=False):
         """
         Generate a source manifest for the system.
         """
@@ -316,9 +316,16 @@ def get_source_manifest(cls):
 
         # Find all source files
         steps_dir = os.path.join(cls.git_dir, 'steps')
-        for file in os.listdir(steps_dir):
-            if os.path.isdir(os.path.join(steps_dir, file)):
-                sourcef = os.path.join(steps_dir, file, "sources")
+        with open(os.path.join(steps_dir, 'manifest'), 'r', encoding="utf_8") as file:
+            for line in file:
+                if pre_network and line.strip().startswith("improve: ") and "network" in line:
+                    break
+
+                if not line.strip().startswith("build: "):
+                    continue
+
+                step = line.split(" ")[1].split("#")[0].strip()
+                sourcef = os.path.join(steps_dir, step, "sources")
                 if os.path.exists(sourcef):
                     # Read sources from the source file
                     with open(sourcef, "r", encoding="utf_8") as sources:
@@ -331,7 +338,9 @@ def get_source_manifest(cls):
                                 # Automatically determine file name based on URL.
                                 file_name = os.path.basename(line[0])
 
-                            manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
+                            manifest_line = f"{line[1]} {directory} {line[0]} {file_name}"
+                            if manifest_line not in manifest_lines:
+                                manifest_lines.append(manifest_line)
 
         return "\n".join(manifest_lines)
 

From 3305f2a41b13a1bac3e23ea0ddee069c7e3909c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 23:08:42 +0100
Subject: [PATCH 07/13] Use manifest to deduce pre-network sources list

No need to maintain a separate pre-network-sources file anymore,
the list is instead derived from the bootstrap manifest via the
source manifest.
---
 lib/generator.py          | 40 ++++++++----------
 source_manifest.py        |  2 +-
 steps/pre-network-sources | 86 ---------------------------------------
 3 files changed, 19 insertions(+), 109 deletions(-)
 delete mode 100644 steps/pre-network-sources

diff --git a/lib/generator.py b/lib/generator.py
index 54e38599..1aec17f6 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -32,6 +32,7 @@ def __init__(self, tmpdir, arch, external_sources,
         self.tmpdir = tmpdir
         self.tmp_dir = tmpdir.path
         self.external_dir = os.path.join(self.tmp_dir, 'external')
+        self.source_manifest = self.get_source_manifest(not self.external_sources)
 
     def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
         """
@@ -100,8 +101,7 @@ def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
 
     def steps(self):
         """Copy in steps."""
-        source_manifest = self.get_source_manifest(not self.external_sources)
-        self.get_packages(source_manifest)
+        self.get_packages()
 
         shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
 
@@ -155,12 +155,10 @@ def distfiles(self):
         """Copy in distfiles"""
         def copy_no_network_distfiles(out):
             # Note that "no disk" implies "no network" for kernel bootstrap mode
-            pre_src_path = os.path.join(self.git_dir, 'steps', 'pre-network-sources')
-            with open(pre_src_path, 'r', encoding="utf-8") as source_list:
-                for file in source_list.readlines():
-                    file = file.strip()
-                    shutil.copy2(os.path.join(self.distfiles_dir, file),
-                                 os.path.join(out, file))
+            for file in self.source_manifest:
+                file = file[3].strip()
+                shutil.copy2(os.path.join(self.distfiles_dir, file),
+                             os.path.join(out, file))
 
         early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
         main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
@@ -298,11 +296,9 @@ def download_file(self, url, directory, file_name):
                 raise requests.HTTPError("Download failed.")
         return abs_file_name
 
-    def get_packages(self, source_manifest):
+    def get_packages(self):
         """Prepare remaining sources"""
-        for line in source_manifest.split("\n"):
-            line = line.strip().split(" ")
-
+        for line in self.source_manifest:
             path = self.download_file(line[2], line[1], line[3])
             self.check_file(path, line[0])
 
@@ -311,7 +307,7 @@ def get_source_manifest(cls, pre_network=False):
         """
         Generate a source manifest for the system.
         """
-        manifest_lines = []
+        entries = []
         directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
 
         # Find all source files
@@ -329,20 +325,20 @@ def get_source_manifest(cls, pre_network=False):
                 if os.path.exists(sourcef):
                     # Read sources from the source file
                     with open(sourcef, "r", encoding="utf_8") as sources:
-                        for line in sources.readlines():
-                            line = line.strip().split(" ")
+                        for source in sources.readlines():
+                            source = source.strip().split(" ")
 
-                            if len(line) > 2:
-                                file_name = line[2]
+                            if len(source) > 2:
+                                file_name = source[2]
                             else:
                                 # Automatically determine file name based on URL.
-                                file_name = os.path.basename(line[0])
+                                file_name = os.path.basename(source[0])
 
-                            manifest_line = f"{line[1]} {directory} {line[0]} {file_name}"
-                            if manifest_line not in manifest_lines:
-                                manifest_lines.append(manifest_line)
+                            entry = (source[1], directory, source[0], file_name)
+                            if entry not in entries:
+                                entries.append(entry)
 
-        return "\n".join(manifest_lines)
+        return entries
 
 stage0_arch_map = {
     "amd64": "AMD64",
diff --git a/source_manifest.py b/source_manifest.py
index 81a1ba96..a9725381 100755
--- a/source_manifest.py
+++ b/source_manifest.py
@@ -13,7 +13,7 @@
 
 def main():
     """Generate a source manifest for a system"""
-    print(Generator.get_source_manifest())
+    print('\n'.join(map(' '.join, Generator.get_source_manifest())))
 
 if __name__ == "__main__":
     main()
diff --git a/steps/pre-network-sources b/steps/pre-network-sources
deleted file mode 100644
index 8b54d0d7..00000000
--- a/steps/pre-network-sources
+++ /dev/null
@@ -1,86 +0,0 @@
-mes-0.25.tar.gz
-nyacc-1.00.2.tar.gz
-tcc-0.9.26.tar.gz
-tcc-0.9.27.tar.bz2
-fiwix-1.4.0-lb3.tar.gz
-lwext4-1.0.0-lb1.tar.gz
-make-3.82.tar.bz2
-patch-2.5.9.tar.gz
-gzip-1.2.4.tar.gz
-tar-1.12.tar.gz
-sed-4.0.9.tar.gz
-bzip2-1.0.8.tar.gz
-coreutils-5.0.tar.bz2
-heirloom-devtools-070527.tar.bz2
-bash-2.05b.tar.gz
-flex-2.5.11.tar.gz
-tcc-0.9.27.tar.bz2
-musl-1.1.24.tar.gz
-tcc-0.9.27.tar.bz2
-musl-1.1.24.tar.gz
-tcc-0.9.27.tar.bz2
-sed-4.0.9.tar.gz
-bzip2-1.0.8.tar.gz
-m4-1.4.7.tar.gz
-flex-2.6.4.tar.gz
-bison-3.4.1.tar.gz
-bison-3.4.1.tar.gz
-bison-3.4.1.tar.gz
-grep-2.4.tar.gz
-diffutils-2.7.tar.gz
-coreutils-5.0.tar.bz2
-coreutils-6.10.tar.gz
-gawk-3.0.4.tar.gz
-perl-5.000.tar.gz
-perl-5.003.tar.gz
-perl5.004_05.tar.gz
-perl5.005_03.tar.gz
-perl-5.6.2.tar.gz
-autoconf-2.52.tar.bz2
-automake-1.6.3.tar.bz2
-automake-1.6.3.tar.bz2
-autoconf-2.53.tar.bz2
-automake-1.7.tar.bz2
-autoconf-2.54.tar.bz2
-autoconf-2.55.tar.bz2
-automake-1.7.8.tar.bz2
-autoconf-2.57.tar.bz2
-autoconf-2.59.tar.bz2
-automake-1.8.5.tar.bz2
-help2man-1.36.4.tar.gz
-autoconf-2.61.tar.bz2
-automake-1.9.6.tar.bz2
-automake-1.10.3.tar.bz2
-autoconf-2.64.tar.bz2
-automake-1.11.2.tar.bz2
-autoconf-2.69.tar.gz
-libtool-2.2.4.tar.bz2
-automake-1.15.1.tar.gz
-binutils-2.30.tar.bz2
-musl-1.1.24.tar.gz
-tcc-0.9.27.tar.bz2
-gcc-core-4.0.4.tar.bz2
-automake-1.16.3.tar.gz
-findutils-4.2.33.tar.gz
-gnulib-8e128e.tar.gz
-musl-1.2.4.tar.gz
-gcc-core-4.0.4.tar.bz2
-automake-1.16.3.tar.gz
-util-linux-2.19.1.tar.gz
-e2fsprogs-1.45.7.tar.gz
-CaseFolding.txt
-DerivedAge.txt
-DerivedCombiningClass.txt
-DerivedCoreProperties.txt
-NormalizationCorrections.txt
-NormalizationTest.txt
-UnicodeData.txt
-v10.0.1.tar.gz
-kbd-1.15.tar.gz
-make-3.82.tar.bz2
-ed-1.4.tar.gz
-bc-1.07.1.tar.gz
-v2.0.22.tar.gz
-linux-4.9.10.tar.gz
-deblob-4.9
-curl-7.88.1.tar.bz2

From 55d3c36e09437d150a2a8876f25daaa3b9281148 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 23:27:02 +0100
Subject: [PATCH 08/13] Move kernel-bootstrap source image out of the directory
 it's generated from

This way, an incomplete version of the image itself won't get included
in srcfs anymore.
---
 lib/generator.py | 2 +-
 rootfs.py        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index 1aec17f6..bd532ad0 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -92,7 +92,7 @@ def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
             shutil.copytree(self.repo_path, repo_dir)
 
         if kernel_bootstrap:
-            self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'), target_size)
+            self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size)
 
         if kernel_bootstrap and (self.external_sources or self.repo_path):
             self.tmpdir.umount_disk('external')
diff --git a/rootfs.py b/rootfs.py
index 172065b8..d202cb25 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -244,7 +244,7 @@ def bootstrap(args, generator, tmpdir, size):
             generator.prepare(kernel_bootstrap=True, target_size=size)
             image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
-            print(f"  1. Take {image_path}/disk.img and write it to a boot drive and then boot it.")
+            print(f"  1. Take {image_path}.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:
@@ -268,7 +268,7 @@ def bootstrap(args, generator, tmpdir, size):
                 '-m', str(args.qemu_ram) + 'M',
                 '-smp', str(args.cores),
                 '-no-reboot',
-                '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw'
+                '-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw'
             ]
             if tmpdir.get_disk("external") is not None:
                 arg_list += [

From 32dc4c702b38718a01088fe856d309346f4e0f8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 23:55:27 +0100
Subject: [PATCH 09/13] More verbose error message on HTTP errors

---
 lib/generator.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/generator.py b/lib/generator.py
index bd532ad0..428a5419 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -293,7 +293,8 @@ def download_file(self, url, directory, file_name):
                 with open(abs_file_name, 'wb') as target_file:
                     target_file.write(response.raw.read())
             else:
-                raise requests.HTTPError("Download failed.")
+                raise requests.HTTPError("Download failed: HTTP " +
+                    response.status_code + " " + response.reason)
         return abs_file_name
 
     def get_packages(self):

From b45e1f81aebdb5c40e154c7d0902a63102b22ebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Mon, 18 Dec 2023 17:22:52 +0100
Subject: [PATCH 10/13] Fix failure with --external-sources

---
 lib/generator.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/generator.py b/lib/generator.py
index 428a5419..f6df8906 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -168,7 +168,6 @@ def copy_no_network_distfiles(out):
             copy_no_network_distfiles(early_distfile_dir)
 
         if self.external_sources:
-            os.mkdir(main_distfile_dir)
             shutil.copytree(self.distfiles_dir, main_distfile_dir)
         else:
             os.mkdir(main_distfile_dir)

From be1333ee8bebb6f064f4062f4d0627764a144ef7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Fri, 22 Dec 2023 12:47:09 +0100
Subject: [PATCH 11/13] Make tmpdir a method parameter of prepare()

Fixes pylint errors.
---
 lib/generator.py | 27 ++++++++++++++-------------
 rootfs.py        | 15 +++++++--------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index f6df8906..9acd0314 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -14,6 +14,7 @@
 import tarfile
 import requests
 
+# pylint: disable=too-many-instance-attributes
 class Generator():
     """
     Class responsible for generating the basic media to be consumed.
@@ -22,25 +23,25 @@ class Generator():
     git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
     distfiles_dir = os.path.join(git_dir, 'distfiles')
 
-    # pylint: disable=too-many-arguments
-    def __init__(self, tmpdir, arch, external_sources,
-                 early_preseed, repo_path):
+    def __init__(self, arch, external_sources, early_preseed, repo_path):
         self.arch = arch
         self.early_preseed = early_preseed
         self.external_sources = external_sources
         self.repo_path = repo_path
-        self.tmpdir = tmpdir
-        self.tmp_dir = tmpdir.path
-        self.external_dir = os.path.join(self.tmp_dir, 'external')
         self.source_manifest = self.get_source_manifest(not self.external_sources)
+        self.tmp_dir = None
+        self.external_dir = None
 
-    def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
+    def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0):
         """
         Prepare basic media of live-bootstrap.
         /steps -- contains steps to be built
         / -- contains seed to allow steps to be built, containing custom
              scripts and stage0-posix
         """
+        self.tmp_dir = tmpdir.path
+        self.external_dir = os.path.join(self.tmp_dir, 'external')
+
         # We use ext3 here; ext4 actually has a variety of extensions that
         # have been added with varying levels of recency
         # Linux 4.9.10 does not support a bunch of them
@@ -56,17 +57,17 @@ def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
             self.tmp_dir = init_path
 
             if self.repo_path or self.external_sources:
-                self.tmpdir.add_disk("external", filesystem="ext3")
-                self.tmpdir.mount_disk("external", "external")
+                tmpdir.add_disk("external", filesystem="ext3")
+                tmpdir.mount_disk("external", "external")
             else:
                 self.external_dir = os.path.join(self.tmp_dir, 'external')
         elif using_kernel:
             self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
-            self.tmpdir.add_disk("disk",
+            tmpdir.add_disk("disk",
                             filesystem="ext3",
                             size=(target_size + "M") if target_size else "16G",
                             bootable=True)
-            self.tmpdir.mount_disk("disk", "disk")
+            tmpdir.mount_disk("disk", "disk")
             self.external_dir = os.path.join(self.tmp_dir, 'external')
 
         os.makedirs(self.external_dir, exist_ok=True)
@@ -95,9 +96,9 @@ def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
             self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size)
 
         if kernel_bootstrap and (self.external_sources or self.repo_path):
-            self.tmpdir.umount_disk('external')
+            tmpdir.umount_disk('external')
         elif using_kernel:
-            self.tmpdir.umount_disk('disk')
+            tmpdir.umount_disk('disk')
 
     def steps(self):
         """Copy in steps."""
diff --git a/rootfs.py b/rootfs.py
index d202cb25..8641d021 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -162,8 +162,7 @@ def check_types():
     if args.tmpfs:
         tmpdir.tmpfs(size=args.tmpfs_size)
 
-    generator = Generator(tmpdir=tmpdir,
-                          arch=args.arch,
+    generator = Generator(arch=args.arch,
                           external_sources=args.external_sources,
                           repo_path=args.repo,
                           early_preseed=args.early_preseed)
@@ -181,7 +180,7 @@ def bootstrap(args, generator, tmpdir, size):
         chroot_binary = run_as_root('python3', '-c', find_chroot,
                                     capture_output=True).stdout.decode().strip()
 
-        generator.prepare(using_kernel=False)
+        generator.prepare(tmpdir, using_kernel=False)
 
         arch = stage0_arch_map.get(args.arch, args.arch)
         init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
@@ -189,7 +188,7 @@ def bootstrap(args, generator, tmpdir, size):
 
     elif args.bwrap:
         if not args.internal_ci or args.internal_ci == "pass1":
-            generator.prepare(using_kernel=False)
+            generator.prepare(tmpdir, using_kernel=False)
 
             arch = stage0_arch_map.get(args.arch, args.arch)
             init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
@@ -235,20 +234,20 @@ def bootstrap(args, generator, tmpdir, size):
 
     elif args.bare_metal:
         if args.kernel:
-            generator.prepare(using_kernel=True, target_size=size)
+            generator.prepare(tmpdir, using_kernel=True, target_size=size)
             image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
             print(f"  1. Take {image_path}/initramfs and your kernel, boot using this.")
             print(f"  2. Take {image_path}/disk.img and put this on a writable storage medium.")
         else:
-            generator.prepare(kernel_bootstrap=True, target_size=size)
+            generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
             image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
             print(f"  1. Take {image_path}.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:
-            generator.prepare(using_kernel=True, target_size=size)
+            generator.prepare(tmpdir, using_kernel=True, target_size=size)
 
             run(args.qemu_cmd,
                 '-enable-kvm',
@@ -262,7 +261,7 @@ def bootstrap(args, generator, tmpdir, size):
                 '-nographic',
                 '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
         else:
-            generator.prepare(kernel_bootstrap=True, target_size=size)
+            generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
             arg_list = [
                 '-enable-kvm',
                 '-m', str(args.qemu_ram) + 'M',

From ab9455f9180494840eaf672cae14f9f11a6f163b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Fri, 22 Dec 2023 12:54:12 +0100
Subject: [PATCH 12/13] Fix warnings in existing code revealed by newer pylint

---
 lib/generator.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index 9acd0314..82ecd1e9 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -109,13 +109,13 @@ def steps(self):
     def stage0_posix(self):
         """Copy in all of the stage0-posix"""
         stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
-        for f in os.listdir(stage0_posix_base_dir):
-            orig = os.path.join(stage0_posix_base_dir, f)
-            to = os.path.join(self.tmp_dir, f)
+        for entry in os.listdir(stage0_posix_base_dir):
+            orig = os.path.join(stage0_posix_base_dir, entry)
+            target = os.path.join(self.tmp_dir, entry)
             if os.path.isfile(orig):
-                shutil.copy2(orig, to)
+                shutil.copy2(orig, target)
             else:
-                shutil.copytree(orig, to)
+                shutil.copytree(orig, target)
 
         arch = stage0_arch_map.get(self.arch, self.arch)
         kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
@@ -125,11 +125,12 @@ def stage0_posix(self):
     def seed(self):
         """Copy in extra seed files"""
         seed_dir = os.path.join(self.git_dir, 'seed')
-        for f in os.listdir(seed_dir):
-            if os.path.isfile(os.path.join(seed_dir, f)):
-                shutil.copy2(os.path.join(seed_dir, f), os.path.join(self.tmp_dir, f))
+        for entry in os.listdir(seed_dir):
+            if os.path.isfile(os.path.join(seed_dir, entry)):
+                shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry))
 
-    def add_fiwix_files(self, file_list_path, dirpath):
+    @staticmethod
+    def add_fiwix_files(file_list_path, dirpath):
         """Add files to the list to populate Fiwix file system"""
         for root, _, filepaths in os.walk(dirpath):
             if 'stage0-posix' in root:
@@ -258,7 +259,8 @@ def create_builder_hex0_disk_image(self, image_file_name, size):
             with open(image_file_name, 'ab') as image_file:
                 image_file.truncate(size * megabyte)
 
-    def check_file(self, file_name, expected_hash):
+    @staticmethod
+    def check_file(file_name, expected_hash):
         """Check hash of downloaded source file."""
         with open(file_name, "rb") as downloaded_file:
             downloaded_content = downloaded_file.read() # read entire file as bytes
@@ -271,7 +273,8 @@ def check_file(self, file_name, expected_hash):
 When in doubt, try deleting the file in question -- it will be downloaded again when running \
 this script the next time")
 
-    def download_file(self, url, directory, file_name):
+    @staticmethod
+    def download_file(url, directory, file_name):
         """
         Download a single source archive.
         """

From 529ea4cae19361acdb2e64b4f667d568ba4b64df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Fri, 22 Dec 2023 13:12:40 +0100
Subject: [PATCH 13/13] Explain significance of build-bash and improve-network
 directives

---
 steps/manifest | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/steps/manifest b/steps/manifest
index 4d475a1d..b1ebe684 100644
--- a/steps/manifest
+++ b/steps/manifest
@@ -16,6 +16,13 @@
 # - jump: jump (usually) to a new kernel, executes a script with that name
 #   eg, jump: fiwix
 #
+# The following directives have special significance:
+# - build directives beginning with "bash" (as well as jumps) trigger the generation of
+#   a new script
+# - the first improve directive containing "network" is used by generator.py to deduce
+#   what source files need to be downloaded in advance (files referenced after that will
+#   be downloaded during bootstrap, unless --external-sources is given)
+#
 # Other features:
 # - predicate; based on variables set in bootstrap.cfg, require for something to execute
 #   must be enclosed in brackets with spaces padded