Split out m4 to start binary bootstrap

Updates #80

lib/bootstrap/binary/README.md

@@ -0,0 +1,6 @@
+# Binary bootstrap
+
+Files in this directory are used to build the bootstrap C compiler and shell for each platform.
+These intentionally use the system's toolchain to produce,
+so you will need to either run `zb serve` with `--sandbox=0`,
+or include the appropriate tools using `--sandbox-path`.

lib/bootstrap/binary/linux.lua

@@ -0,0 +1,24 @@
+-- Copyright 2025 The zb Authors
+-- SPDX-License-Identifier: MIT
+
+local function forArchitecture(arch)
+  local m4 = import("../m4.lua") {
+    bash = "/usr";
+    PATH = os.getenv("PATH");
+    ARCH = arch;
+    CC = "gcc";
+    AR = "ar";
+    CFLAGS = "-Wno-implicit-function-declaration";
+  }
+
+  return m4
+end
+
+return setmetatable({}, {
+  __index = function(_, k)
+    for arch, zbArch in pairs(import("../steps.lua").archToZBTable) do
+      if zbArch == k then return forArchitecture(arch) end
+    end
+    return nil
+  end;
+})

lib/bootstrap/bootstrap.lua

@@ -5,72 +5,6 @@ local stage0 <const> = dofile("stage0-posix/x86_64-linux.lua")
 
 local boot <const> = {}
 
-local gnuMirrors <const> = {
-  "https://mirrors.kernel.org/gnu/",
-  "https://ftp.gnu.org/gnu/",
-}
-
-local badGNUURLs <const> = {
-  -- Nix's fetchurl seems to un-lzma tarballs from mirrors.kernel.org.
-  -- Unclear why.
-  "https://mirrors.kernel.org/gnu/coreutils/coreutils-6.10.tar.lzma",
-  "https://mirrors.kernel.org/gnu/libtool/libtool-2.2.4.tar.lzma",
-}
-
----@param args {path: string, hash: string}
-local function fetchGNU(args)
-  for _, mirror in ipairs(gnuMirrors) do
-    local url = mirror..args.path
-    if not table.elem(url, badGNUURLs) then
-      return fetchurl({
-        url = url;
-        hash = args.hash;
-      })
-    end
-  end
-end
-
----Construct a Unix-style search path by appending `subDir`
----to the specified `output` of each of the packages.
----@param output string
----@param subDir string
----@param paths derivation[]
----@return string
-local function makeSearchPathOutput(output, subDir, paths)
-  local parts = {}
-  for i, x in ipairs(paths) do
-    local xout = x[output]
-    if xout then
-      if #parts > 0 then
-        parts[#parts + 1] = ":"
-      end
-      parts[#parts + 1] = tostring(xout)
-      parts[#parts + 1] = "/"
-      parts[#parts + 1] = subDir
-    end
-  end
-  return table.concat(parts)
-end
-
----Construct a binary search path (such as `$PATH`)
----containing the binaries for a set of packages.
----@param pkgs derivation[]
----@return string # colon-separated paths
-local function mkBinPath(pkgs)
-  return makeSearchPathOutput("out", "bin", pkgs)
-end
-
----@param pkgs derivation[]
----@return string
-local function mkIncludePath(pkgs)
-  return makeSearchPathOutput("out", "include", pkgs)
-end
-
----@param pkgs derivation[]
----@return string
-local function mkLibraryPath(pkgs)
-  return makeSearchPathOutput("out", "lib", pkgs)
-end
 
 ---@param args table
 local function kaemDerivation(args)
@@ -90,20 +24,6 @@ local function kaemDerivation(args)
   return derivation(actualArgs)
 end
 
----@param pname string
----@param version string?
----@return string
-local function stepPath(pname, version)
-  local name = pname
-  if version then
-    name = name.."-"..version
-  end
-  return path {
-    name = "live-bootstrap-steps-"..name;
-    path = "live-bootstrap/steps/"..name;
-  }
-end
-
 --- Issue cp commands for the directory.
 ---@param manifest string[]
 ---@return string

lib/bootstrap/live-bootstrap/steps/m4-1.4.7/mk/main.mk

@@ -3,10 +3,10 @@
 
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-CC      = tcc
-AR      = tcc -ar
+CC      ?= tcc
+AR      ?= tcc -ar
 
-CFLAGS  = -I lib \
+CFLAGS  += -I lib \
           -DVERSION=\"1.4.7\" \
           -DPACKAGE_BUGREPORT=\"[email protected]\" \
           -DPACKAGE_STRING=\"GNU\ M4\ 1.4.7\" \

lib/bootstrap/m4.lua

@@ -0,0 +1,38 @@
+-- Copyright 2025 The zb Authors
+-- SPDX-License-Identifier: MIT
+
+local fetchGNU <const> = import "../fetchgnu.lua"
+local steps <const> = import "./steps.lua"
+local strings <const> = import "../strings.lua"
+local tables <const> = import "../tables.lua"
+
+return function(args)
+  return steps.bash(tables.update(
+    {
+      pname = "m4";
+      version = "1.4.7";
+      builder = args.bash.."/bin/bash";
+
+      PATH = args.PATH or strings.mkBinPath {
+        args.tcc,
+        args.bash,
+        args.coreutils,
+        args.sed,
+        args.tar,
+        args.gzip,
+        args.bzip2,
+        args.patch,
+        args.make,
+        args.stage0,
+      };
+
+      tarballs = {
+        fetchGNU {
+          path = "m4/m4-1.4.7.tar.bz2";
+          hash = "sha256:a88f3ddaa7c89cf4c34284385be41ca85e9135369c333fdfa232f3bf48223213";
+        },
+      };
+    },
+    args
+  ))
+end

lib/bootstrap/steps.lua

@@ -0,0 +1,114 @@
+-- Copyright 2025 The zb Authors
+-- SPDX-License-Identifier: MIT
+
+local gpath <const> = path
+local helpers <const> = gpath "live-bootstrap/steps/helpers.sh"
+local helpers_nix <const> = gpath "live-bootstrap/steps/helpers-nix.sh"
+
+archToZBTable = {
+  amd64 = "x86_64";
+  aarch64 = "aarch64";
+}
+
+---@param pname string
+---@param version string?
+---@return string
+function path(pname, version)
+  local name = pname
+  if version then
+    name = name.."-"..version
+  end
+  return gpath {
+    name = "live-bootstrap-steps-"..name;
+    path = "live-bootstrap/steps/"..name;
+  }
+end
+
+---@class bashStepArgs
+---@field pname string
+---@field version string
+---@field builder string
+---@field setup string?
+---@field tarballs derivation[]
+---@field [string] any
+
+---@param args bashStepArgs
+function bash(args)
+  if not args.builder then
+    error("missing builder from steps.bash args", 2)
+  end
+  local pname <const> = args.pname
+  local version <const> = args.version
+  local defaultName <const> = pname.."-"..version
+  local actualArgs <const> = {
+    name = defaultName;
+    system = "x86_64-linux";
+
+    OPERATING_SYSTEM = "Linux";
+    ARCH = "amd64";
+    MAKEJOBS = "-j1";
+    SOURCE_DATE_EPOCH = 0;
+    KBUILD_BUILD_TIMESTAMP = "@0";
+    pkg = defaultName;
+    revision = 0;
+  }
+  if args.ARCH then
+    local a = archToZBTable[args.ARCH]
+    if a then actualArgs.system = a.."-linux" end
+  end
+  for k, v in pairs(args) do
+    if k ~= "setup" then
+      actualArgs[k] = v
+    end
+  end
+  actualArgs.SHELL = actualArgs.builder
+
+  ---@type (string|number|derivation)[]
+  local scriptChunks <const> = {
+    "\z
+      #!/usr/bin/env bash\n\z
+      set -e\n\z
+      export PREFIX=${out}\n\z
+      export DESTDIR=''\n\z
+      export BINDIR=${PREFIX}/bin\n\z
+      export LIBDIR=${PREFIX}/lib\n\z
+      PATH=${BINDIR}:${PATH}\n\z
+      mkdir ${PREFIX}\n",
+    ". ",
+    helpers,
+    "\n",
+    ". ",
+    helpers_nix,
+    "\n",
+    args.setup or "",
+    "\n\z
+      DISTFILES=${TEMPDIR}/distfiles\n\z
+      mkdir ${DISTFILES}\n\z
+      _select() {\n\z
+        local i=\"$1\"\n\z
+        shift\n\z
+        eval \"echo \\${$i}\"\n\z
+      }\n",
+  }
+  for i, t in ipairs(args.tarballs) do
+    scriptChunks[#scriptChunks + 1] = "cp \"$(_select "
+    scriptChunks[#scriptChunks + 1] = i
+    scriptChunks[#scriptChunks + 1] = " $tarballs)\" ${DISTFILES}/"
+    ---@diagnostic disable-next-line: param-type-mismatch
+    scriptChunks[#scriptChunks + 1] = baseNameOf(t.name)
+    scriptChunks[#scriptChunks + 1] = "\n"
+  end
+  scriptChunks[#scriptChunks + 1] = "\z
+      unset _select\n\z
+      SRCDIR=${TEMPDIR}/src\n\z
+      mkdir ${SRCDIR}\n\z
+      cp -R "
+  scriptChunks[#scriptChunks + 1] = path(actualArgs.pkg)
+  scriptChunks[#scriptChunks + 1] = " ${SRCDIR}/${pkg}\n\z
+      chmod -R +w ${SRCDIR}/${pkg}\n\z
+      build ${pkg}\n"
+  local script <const> = table.concat(scriptChunks)
+  actualArgs.args = { toFile(actualArgs.name.."-builder.sh", script) }
+
+  return derivation(actualArgs)
+end

lib/fetchgnu.lua

@@ -0,0 +1,27 @@
+-- Copyright 2025 The zb Authors
+-- SPDX-License-Identifier: MIT
+
+local gnuMirrors <const> = {
+  "https://mirrors.kernel.org/gnu/",
+  "https://ftp.gnu.org/gnu/",
+}
+
+local badGNUURLs <const> = {
+  -- Nix's fetchurl seems to un-lzma tarballs from mirrors.kernel.org.
+  -- Unclear why.
+  "https://mirrors.kernel.org/gnu/coreutils/coreutils-6.10.tar.lzma",
+  "https://mirrors.kernel.org/gnu/libtool/libtool-2.2.4.tar.lzma",
+}
+
+---@param args {path: string, hash: string}
+return function(args)
+  for _, mirror in ipairs(gnuMirrors) do
+    local url = mirror..args.path
+    if not table.elem(url, badGNUURLs) then
+      return fetchurl({
+        url = url;
+        hash = args.hash;
+      })
+    end
+  end
+end

lib/strings.lua

@@ -0,0 +1,44 @@
+-- Copyright 2025 The zb Authors
+-- SPDX-License-Identifier: MIT
+
+---Construct a Unix-style search path by appending `subDir`
+---to the specified `output` of each of the packages.
+---@param output string
+---@param subDir string
+---@param paths derivation[]
+---@return string
+function makeSearchPathOutput(output, subDir, paths)
+  local parts = {}
+  for i, x in ipairs(paths) do
+    local xout = x[output]
+    if xout then
+      if #parts > 0 then
+        parts[#parts + 1] = ":"
+      end
+      parts[#parts + 1] = tostring(xout)
+      parts[#parts + 1] = "/"
+      parts[#parts + 1] = subDir
+    end
+  end
+  return table.concat(parts)
+end
+
+---Construct a binary search path (such as `$PATH`)
+---containing the binaries for a set of packages.
+---@param pkgs derivation[]
+---@return string # colon-separated paths
+function mkBinPath(pkgs)
+  return makeSearchPathOutput("out", "bin", pkgs)
+end
+
+---@param pkgs derivation[]
+---@return string
+function mkIncludePath(pkgs)
+  return makeSearchPathOutput("out", "include", pkgs)
+end
+
+---@param pkgs derivation[]
+---@return string
+function mkLibraryPath(pkgs)
+  return makeSearchPathOutput("out", "lib", pkgs)
+end

lib/tables.lua

@@ -0,0 +1,17 @@
+-- Copyright 2025 The zb Authors
+-- SPDX-License-Identifier: MIT
+
+---Copy the pairs from each argument into the first argument.
+---@generic T: table
+---@param t T
+---@param ... table
+---@return T
+function update(t, ...)
+  local n <const> = select("#", ...)
+  for i = 1, n do
+    for k, v in pairs(select(i, ...)) do
+      t[k] = v
+    end
+  end
+  return t
+end