feat: Refactor init into test-distro

The init module contains a small init system for running our integration
tests against a kernel. While we don't need a full-blown linux distro,
we do need some utilities.

Once such utility is `modprobe` which allows us to load kernel modules.
Rather than create a new module for this utility, I've instead
refactored `init` into `test-distro` which is a module that contains
multiple binaries.

The xtask code has been adjusted to ensure these binaries are inserted
into the correct places in our cpio archive, as well as bringing in the
kernel modules.

Signed-off-by: Dave Tucker <[email protected]>

Author: dave-tucker

.github/scripts/find_kernels.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import os
+import glob
+import sys
+from typing import List
+
+def find_kernels(directory: str) -> List[str]:
+    return glob.glob(f"{directory}/**/vmlinuz-*", recursive=True)
+
+def find_modules_directory(directory: str, kernel: str) -> str:
+    matches = glob.glob(f"{directory}/**/modules/{kernel}", recursive=True)
+    if len(matches) != 1:
+        raise Exception(f"Expected to find exactly one modules directory. Found {len(matches)}.")
+    return matches[0]
+
+def main() -> None:
+    try:
+        images = find_kernels('test/.tmp')
+        modules = []
+
+        for image in images:
+            image_name = os.path.basename(image).replace('vmlinuz-', '')
+            module_dir = find_modules_directory('test/.tmp', image_name)
+            modules.append(module_dir)
+
+        args = ' '.join(f"{image}:{module}" for image, module in zip(images, modules))
+        print(args)
+    except Exception as e:
+        print(e)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

.github/workflows/ci.yml

@@ -51,8 +51,8 @@ jobs:
         if: github.event_name != 'pull_request' && github.repository_owner == 'aya-rs'
         with:
           branch: create-pull-request/public-api
-          commit-message: 'public-api: regenerate'
-          title: 'public-api: regenerate'
+          commit-message: "public-api: regenerate"
+          title: "public-api: regenerate"
           body: |
             **Automated changes**
 
@@ -228,7 +228,7 @@ jobs:
         run: |
           set -euxo pipefail
           sudo apt update
-          sudo apt -y install lynx qemu-system-{arm,x86}
+          sudo apt -y install lynx qemu-system-{arm,x86} musl-tools
           echo /usr/lib/llvm-15/bin >> $GITHUB_PATH
 
       - name: Install prerequisites
@@ -240,6 +240,10 @@ jobs:
         # The tar shipped on macOS doesn't support --wildcards, so we need GNU tar.
         #
         # The clang shipped on macOS doesn't support BPF, so we need LLVM from brew.
+        #
+        # We need a musl C toolchain to compile our `test-distro` since some of
+        # our dependencies have build scripts that compile C code (i.e xz2).
+        # This is provided by `brew install filosottile/musl-cross/musl-cross`.
         run: |
           set -euxo pipefail
           brew update
@@ -250,6 +254,8 @@ jobs:
           echo $(brew --prefix curl)/bin >> $GITHUB_PATH
           echo $(brew --prefix gnu-tar)/libexec/gnubin >> $GITHUB_PATH
           echo $(brew --prefix llvm)/bin >> $GITHUB_PATH
+          brew install filosottile/musl-cross/musl-cross
+          ln -s "$(brew --prefix musl-cross)/bin/x86_64-linux-musl-gcc" /usr/local/bin/musl-gcc
 
       - uses: dtolnay/rust-toolchain@nightly
         with:
@@ -302,21 +308,46 @@ jobs:
         # TODO: enable tests on kernels before 6.0.
         run: .github/scripts/download_kernel_images.sh test/.tmp/debian-kernels/amd64 amd64 6.1 6.10
 
+      - name: Cleanup stale kernels and modules
+        run: |
+          set -euxo pipefail
+          rm -rf test/.tmp/boot test/.tmp/lib
+
       - name: Extract debian kernels
         run: |
           set -euxo pipefail
+          # The wildcard '**/boot/*' extracts kernel images and config.
+          # The wildcard '**/modules/*' extracts kernel modules.
+          # Modules are required since not all parts of the kernel we want to
+          # test are built-in.
           find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \
-            sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -"
+            sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp \
+              --wildcards --extract '**/boot/*' '**/modules/*' --file -"
 
       - name: Run local integration tests
         if: runner.os == 'Linux'
         run: cargo xtask integration-test local
 
       - name: Run virtualized integration tests
+        if: runner.os == 'Linux'
+        run: |
+          set -euxo pipefail
+          ARGS=$(./.github/scripts/find_kernels.py)
+          cargo xtask integration-test vm --cache-dir test/.tmp \
+            --github-api-token ${{ secrets.GITHUB_TOKEN }} \
+            ${ARGS}
+
+      - name: Run virtualized integration tests
+        if: runner.os == 'macOS'
+        env:
+          # This sets the linker to the one installed by FiloSottile/musl-cross.
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: x86_64-linux-musl-gcc
         run: |
           set -euxo pipefail
-          find test/.tmp -name 'vmlinuz-*' -print0 | xargs -t -0 \
-            cargo xtask integration-test vm --cache-dir test/.tmp --github-api-token ${{ secrets.GITHUB_TOKEN }}
+          ARGS=$(./.github/scripts/find_kernels.py)
+          cargo xtask integration-test vm --cache-dir test/.tmp \
+            --github-api-token ${{ secrets.GITHUB_TOKEN }} \
+            ${ARGS}
 
   # Provides a single status check for the entire build workflow.
   # This is used for merge automation, like Mergify, since GH actions

Cargo.toml

@@ -7,7 +7,7 @@ members = [
     "aya-log-parser",
     "aya-obj",
     "aya-tool",
-    "init",
+    "test-distro",
     "test/integration-common",
     "test/integration-test",
     "xtask",
@@ -33,7 +33,7 @@ default-members = [
     "aya-log-parser",
     "aya-obj",
     "aya-tool",
-    "init",
+    "test-distro",
     "test/integration-common",
     # test/integration-test is omitted; including it in this list causes `cargo test` to run its
     # tests, and that doesn't work unless they've been built with `cargo xtask`.
@@ -74,6 +74,7 @@ diff = { version = "0.1.13", default-features = false }
 env_logger = { version = "0.11", default-features = false }
 epoll = { version = "4.3.3", default-features = false }
 futures = { version = "0.3.28", default-features = false }
+glob = { version = "0.3.0", default-features = false }
 hashbrown = { version = "0.15.0", default-features = false }
 indoc = { version = "2.0", default-features = false }
 libc = { version = "0.2.105", default-features = false }
@@ -101,8 +102,10 @@ test-log = { version = "0.2.13", default-features = false }
 testing_logger = { version = "0.1.1", default-features = false }
 thiserror = { version = "2.0.3", default-features = false }
 tokio = { version = "1.24.0", default-features = false }
+walkdir = { version = "2", default-features = false }
 which = { version = "7.0.0", default-features = false }
 xdpilone = { version = "1.0.5", default-features = false }
+xz2 = { version = "0.1.7", default-features = false }
 
 [workspace.lints.rust]
 unused-extern-crates = "warn"

init/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "test-distro"
+publish = false
+version = "0.1.0"
+
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[[bin]]
+name = "init"
+path = "src/init.rs"
+
+[[bin]]
+name = "modprobe"
+path = "src/modprobe.rs"
+
+[[bin]]
+name = "depmod"
+path = "src/depmod.rs"
+
+[dependencies]
+anyhow = { workspace = true, features = ["std"] }
+clap = { workspace = true, default-features = true, features = ["derive"] }
+glob = { workspace = true }
+nix = { workspace = true, features = [
+    "user",
+    "fs",
+    "mount",
+    "reboot",
+    "kmod",
+    "feature",
+] }
+object = { workspace = true, features = ["elf", "read_core", "std"] }
+walkdir = { workspace = true }
+xz2 = { workspace = true }

test-distro/Cargo.toml

@@ -0,0 +1,131 @@
+//! depmod is used to build the modules.alias file to assist with loading
+//! kernel modules.
+//!
+//! This implementation is incredibly naive and is only designed to work within
+//! the constraints of the test environment. Not for production use.
+
+use std::{
+    fs::File,
+    io::{BufWriter, Read, Write as _},
+    path::PathBuf,
+};
+
+use anyhow::{Context as _, anyhow};
+use clap::Parser;
+use object::{Object, ObjectSection, ObjectSymbol};
+use test_distro::resolve_modules_dir;
+use walkdir::WalkDir;
+use xz2::read::XzDecoder;
+
+#[derive(Parser)]
+struct Args {
+    #[clap(long, short)]
+    base_dir: Option<PathBuf>,
+}
+
+fn main() -> anyhow::Result<()> {
+    let Args { base_dir } = Parser::parse();
+
+    let modules_dir = if let Some(base_dir) = base_dir {
+        base_dir
+    } else {
+        resolve_modules_dir().context("failed to resolve modules dir")?
+    };
+
+    let modules_alias = modules_dir.join("modules.alias");
+    let f = std::fs::OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open(&modules_alias)
+        .with_context(|| format!("failed to open: {}", modules_alias.display()))?;
+    let mut output = BufWriter::new(&f);
+    for entry in WalkDir::new(modules_dir) {
+        let entry = entry.context("failed to read entry in walkdir")?;
+        if entry.file_type().is_file() {
+            let path = entry.path();
+
+            if path
+                .extension()
+                .is_none_or(|ext| ext != "ko" && ext != "xz")
+            {
+                continue;
+            }
+
+            let module_name = path
+                .file_name()
+                .ok_or_else(|| anyhow!("{} does not have a file name", path.display()))?
+                .to_str()
+                .ok_or_else(|| anyhow!("{} is not valid utf-8", path.display()))?;
+
+            let (module_name, compressed) =
+                if let Some(module_name) = module_name.strip_suffix(".xz") {
+                    (module_name, true)
+                } else {
+                    (module_name, false)
+                };
+
+            let module_name = module_name
+                .strip_suffix(".ko")
+                .ok_or_else(|| anyhow!("{} does not end in .ko", path.display()))?;
+
+            let mut f =
+                File::open(path).with_context(|| format!("failed to open: {}", path.display()))?;
+            let stat = f
+                .metadata()
+                .with_context(|| format!("failed to get metadata for {}", path.display()))?;
+
+            if compressed {
+                let mut decoder = XzDecoder::new(f);
+                // We don't know the size of the decompressed data, so we assume it's
+                // no more than twice the size of the compressed data.
+                let mut decompressed = Vec::with_capacity(stat.len() as usize * 2);
+                decoder.read_to_end(&mut decompressed)?;
+                read_aliases_from_module(&decompressed, module_name, &mut output)
+            } else {
+                let mut buf = Vec::with_capacity(stat.len() as usize);
+                f.read_to_end(&mut buf)
+                    .with_context(|| format!("failed to read: {}", path.display()))?;
+                read_aliases_from_module(&buf, module_name, &mut output)
+            }
+            .with_context(|| format!("failed to read aliases from module {}", path.display()))?;
+        }
+    }
+    Ok(())
+}
+
+fn read_aliases_from_module(
+    contents: &[u8],
+    module_name: &str,
+    output: &mut BufWriter<&File>,
+) -> Result<(), anyhow::Error> {
+    let obj = object::read::File::parse(contents).context("failed to parse")?;
+
+    let (section_idx, data) = obj
+        .sections()
+        .find_map(|s| {
+            s.name()
+                .is_ok_and(|name| name == ".modinfo")
+                .then(|| s.data().ok())
+                .flatten()
+                .map(|data| (s.index(), data))
+        })
+        .context("no .modinfo section")?;
+
+    for s in obj.symbols() {
+        let name = s.name().context("failed to get symbol name")?;
+        if name.contains("alias") && s.section_index() == Some(section_idx) {
+            let start = s.address() as usize;
+            let end = start + s.size() as usize;
+            let sym_data = &data[start..end];
+            let cstr = std::ffi::CStr::from_bytes_with_nul(sym_data)
+                .context("failed to convert to cstr")?;
+            let sym_str = cstr.to_str().context("failed to convert to str")?;
+            let alias = sym_str
+                .strip_prefix("alias=")
+                .context("failed to strip prefix")?;
+            writeln!(output, "alias {} {}", alias, module_name).expect("write");
+        }
+    }
+    Ok(())
+}

test-distro/src/depmod.rs

@@ -57,6 +57,14 @@ fn run() -> anyhow::Result<()> {
             data: None,
             target_mode: Some(RXRXRX),
         },
+        Mount {
+            source: "dev",
+            target: "/dev",
+            fstype: "devtmpfs",
+            flags: nix::mount::MsFlags::empty(),
+            data: None,
+            target_mode: None,
+        },
         Mount {
             source: "sysfs",
             target: "/sys",

init/src/main.rs renamed to test-distro/src/init.rs

@@ -0,0 +1,30 @@
+use std::path::PathBuf;
+
+use anyhow::Context as _;
+use nix::sys::utsname::uname;
+
+/// Kernel modules are in `/lib/modules`.
+/// They may be in the root of this directory,
+/// or in subdirectory named after the kernel release.
+pub fn resolve_modules_dir() -> anyhow::Result<PathBuf> {
+    let modules_dir = PathBuf::from("/lib/modules");
+    let stat = modules_dir
+        .metadata()
+        .with_context(|| format!("stat(): {}", modules_dir.display()))?;
+    if stat.is_dir() {
+        return Ok(modules_dir);
+    }
+
+    let utsname = uname().context("uname()")?;
+    let release = utsname.release();
+    let modules_dir = modules_dir.join(release);
+    let stat = modules_dir
+        .metadata()
+        .with_context(|| format!("stat(): {}", modules_dir.display()))?;
+    anyhow::ensure!(
+        stat.is_dir(),
+        "{} is not a directory",
+        modules_dir.display()
+    );
+    Ok(modules_dir)
+}