feat: Refactor init into test-distro

The init module contains a small init system for running our integration
tests against a kernel. While we don't need a full-blown linux distro,
we do need some utilities.

Once such utility is `modprobe` which allows us to load kernel modules.
Rather than create a new module for this utility, I've instead
refactored `init` into `test-distro` which is a module that contains
multiple binaries.

The xtask code has been adjusted to ensure these binaries are inserted
into the correct places in our cpio archive, as well as bringing in the
kernel modules.

Signed-off-by: Dave Tucker <[email protected]>

Author: dave-tucker

.github/scripts/find_kernels.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+import os
+import glob
+import sys
+from typing import List
+
+def find_kernels(directory: str) -> List[str]:
+    return glob.glob(f"{directory}/**/vmlinuz-*", recursive=True)
+
+def find_modules_directory(directory: str, kernel: str) -> str:
+    matches = glob.glob(f"{directory}/**/modules/{kernel}", recursive=True)
+    if len(matches) == 0:
+        print(f"ERROR! No modules directory found for kernel {kernel}")
+        sys.exit(1)
+    return matches[0]
+
+def main() -> None:
+    images = find_kernels('test/.tmp')
+    modules = []
+
+    for image in images:
+        image_name = os.path.basename(image).replace('vmlinuz-', '')
+        module_dir = find_modules_directory('test/.tmp', image_name)
+        modules.append(module_dir[0])
+        
+    if len(images) != len(modules):
+        print(f"IMAGES={images}")
+        print(f"MODULES={modules}")
+        print("ERROR! len images != len modules")
+        sys.exit(1)
+
+    args = ' '.join(f"-i {image} -m {module}" for image, module in zip(images, modules))
+    print(args)
+
+if __name__ == "__main__":
+    main()

.github/workflows/ci.yml

@@ -224,7 +224,7 @@ jobs:
         run: |
           set -euxo pipefail
           sudo apt update
-          sudo apt -y install lynx qemu-system-{arm,x86}
+          sudo apt -y install lynx qemu-system-{arm,x86} musl-tools
           echo /usr/lib/llvm-15/bin >> $GITHUB_PATH
 
       - name: Install prerequisites
@@ -236,6 +236,13 @@ jobs:
         # The tar shipped on macOS doesn't support --wildcards, so we need GNU tar.
         #
         # The clang shipped on macOS doesn't support BPF, so we need LLVM from brew.
+        #
+        # We need a musl C toolchain to compile our `test-distro` since some of
+        # our dependencies have build scripts that compile C code (i.e xz2).
+        # This is provided by `brew install filosottile/musl-cross/musl-cross`.
+        # Setting the linker for x86_64-unknown-linux-musl to x86_64-linux-musl-gcc
+        # breaks compilation on linux. The simplest option to solve this is to
+        # symlink x86_64-linux-musl-gcc to musl-gcc.
         run: |
           set -euxo pipefail
           brew update
@@ -246,6 +253,8 @@ jobs:
           echo $(brew --prefix curl)/bin >> $GITHUB_PATH
           echo $(brew --prefix gnu-tar)/libexec/gnubin >> $GITHUB_PATH
           echo $(brew --prefix llvm)/bin >> $GITHUB_PATH
+          brew install filosottile/musl-cross/musl-cross
+          ln -s "$(brew --prefix musl-cross)/bin/x86_64-linux-musl-gcc" /usr/local/bin/musl-gcc
 
       - uses: dtolnay/rust-toolchain@nightly
         with:
@@ -303,8 +312,15 @@ jobs:
       - name: Extract debian kernels
         run: |
           set -euxo pipefail
+          # Remove old images and modules.
+          rm -rf test/.tmp/boot test/.tmp/lib
+          # The wildcard '**/boot/*' extracts kernel images and config.
+          # The wildcard '**/modules/*' extracts kernel modules.
+          # Modules are required since not all parts of the kernel we want to
+          # test are built-in.
           find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \
-            sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -"
+            sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp \
+              --wildcards --extract '**/boot/*' '**/modules/*' --file -"
 
       - name: Run local integration tests
         if: runner.os == 'Linux'
@@ -313,8 +329,10 @@ jobs:
       - name: Run virtualized integration tests
         run: |
           set -euxo pipefail
-          find test/.tmp -name 'vmlinuz-*' -print0 | xargs -t -0 \
-            cargo xtask integration-test vm --cache-dir test/.tmp --github-api-token ${{ secrets.GITHUB_TOKEN }}
+          ARGS=$(./.github/scripts/find_kernels.py)
+          cargo xtask integration-test vm --cache-dir test/.tmp \
+            --github-api-token ${{ secrets.GITHUB_TOKEN }} \
+            ${ARGS}
 
   # Provides a single status check for the entire build workflow.
   # This is used for merge automation, like Mergify, since GH actions

Cargo.toml

@@ -7,7 +7,7 @@ members = [
     "aya-log-parser",
     "aya-obj",
     "aya-tool",
-    "init",
+    "test-distro",
     "test/integration-common",
     "test/integration-test",
     "xtask",
@@ -33,7 +33,7 @@ default-members = [
     "aya-log-parser",
     "aya-obj",
     "aya-tool",
-    "init",
+    "test-distro",
     "test/integration-common",
     # test/integration-test is omitted; including it in this list causes `cargo test` to run its
     # tests, and that doesn't work unless they've been built with `cargo xtask`.
@@ -73,6 +73,7 @@ diff = { version = "0.1.13", default-features = false }
 env_logger = { version = "0.11", default-features = false }
 epoll = { version = "4.3.3", default-features = false }
 futures = { version = "0.3.28", default-features = false }
+glob = { version = "0.3.0", default-features = false }
 hashbrown = { version = "0.15.0", default-features = false }
 indoc = { version = "2.0", default-features = false }
 libc = { version = "0.2.105", default-features = false }
@@ -99,8 +100,10 @@ test-log = { version = "0.2.13", default-features = false }
 testing_logger = { version = "0.1.1", default-features = false }
 thiserror = { version = "2.0.3", default-features = false }
 tokio = { version = "1.24.0", default-features = false }
+walkdir = { version = "2", default-features = false }
 which = { version = "7.0.0", default-features = false }
 xdpilone = { version = "1.0.5", default-features = false }
+xz2 = { version = "0.1.7", default-features = false }
 
 [profile.release.package.integration-ebpf]
 debug = 2

init/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "test-distro"
+version = "0.1.0"
+publish = false
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+edition.workspace = true
+
+[[bin]]
+name = "init"
+path = "src/init.rs"
+
+[[bin]]
+name = "modprobe"
+path = "src/modprobe.rs"
+
+[[bin]]
+name = "depmod"
+path = "src/depmod.rs"
+
+[dependencies]
+anyhow = { workspace = true, features = ["std"] }
+object = { workspace = true, features = ["elf", "read_core", "std"] }
+clap = { workspace = true, default-features = true, features = ["derive"] }
+nix = { workspace = true, features = [
+    "user",
+    "fs",
+    "mount",
+    "reboot",
+    "kmod",
+    "feature",
+] }
+glob = { workspace = true }
+xz2 = { workspace = true }
+walkdir = { workspace = true }

test-distro/Cargo.toml

@@ -0,0 +1,122 @@
+//! depmod is used to build the modules.alias file to assist with loading
+//! kernel modules.
+//!
+//! This implementation is incredibly naive and is only designed to work within
+//! the constraints of the test environment. Not for production use.
+
+use std::{
+    fs::File,
+    io::{BufWriter, Read, Write as _},
+    path::PathBuf,
+};
+
+use anyhow::Context as _;
+use clap::Parser;
+use object::{Object, ObjectSection, ObjectSymbol};
+use test_distro::resolve_modules_dir;
+use walkdir::WalkDir;
+use xz2::read::XzDecoder;
+
+#[derive(Parser)]
+struct Args {
+    #[clap(long, short)]
+    base_dir: Option<PathBuf>,
+}
+
+fn main() -> anyhow::Result<()> {
+    let Args { base_dir } = Parser::parse();
+
+    let modules_dir = if let Some(base_dir) = base_dir {
+        base_dir
+    } else {
+        resolve_modules_dir().context("Failed to resolve modules dir")?
+    };
+
+    let modules_alias = modules_dir.join("modules.alias");
+    let f = std::fs::OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open(&modules_alias)
+        .with_context(|| format!("failed to open: {}", modules_alias.display()))?;
+    let mut output = BufWriter::new(&f);
+    for entry in WalkDir::new(modules_dir) {
+        let entry = entry.context("failed to read entry in walkdir")?;
+        if entry.file_type().is_file() {
+            let path = entry.path();
+            if let Some(extension) = path.extension() {
+                if extension != "ko" && extension != "xz" {
+                    continue;
+                }
+                let module_name = path
+                    .file_stem()
+                    .ok_or(anyhow::anyhow!("failed to get file stem"))?
+                    .to_os_string()
+                    .into_string()
+                    .map_err(|_| anyhow::anyhow!("failed to convert to string"))?
+                    .replace(".ko", "");
+                let mut f = File::open(path)
+                    .with_context(|| format!("failed to open: {}", path.display()))?;
+                let stat = f
+                    .metadata()
+                    .with_context(|| format!("Failed to get metadata for {}", path.display()))?;
+                if extension == "xz" {
+                    let mut decoder = XzDecoder::new(f);
+                    let mut decompressed = Vec::with_capacity(stat.len() as usize * 2);
+                    decoder.read_to_end(&mut decompressed)?;
+                    read_aliases_from_module(&decompressed, &module_name, &mut output)
+                } else {
+                    let mut buf = Vec::with_capacity(stat.len() as usize);
+                    f.read_to_end(&mut buf)
+                        .with_context(|| format!("Failed to read: {}", path.display()))?;
+                    read_aliases_from_module(&buf, &module_name, &mut output)
+                }
+                .with_context(|| {
+                    format!("Failed to read aliases from module {}", path.display())
+                })?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+fn read_aliases_from_module(
+    contents: &[u8],
+    module_name: &str,
+    output: &mut BufWriter<&File>,
+) -> Result<(), anyhow::Error> {
+    let obj = object::read::File::parse(contents).context("not an object file")?;
+
+    let (section_idx, data) = obj
+        .sections()
+        .filter_map(|s| {
+            if let Ok(name) = s.name() {
+                if name == ".modinfo" {
+                    if let Ok(data) = s.data() {
+                        return Some((s.index(), data));
+                    }
+                }
+            }
+            None
+        })
+        .next()
+        .context("no .modinfo section")?;
+
+    obj.symbols()
+        .try_for_each(|s| -> Result<(), anyhow::Error> {
+            let name = s.name().context("failed to get symbol name")?;
+            if name.contains("alias") && s.section_index() == Some(section_idx) {
+                let start = s.address() as usize;
+                let end = start + s.size() as usize;
+                let sym_data = &data[start..end];
+                let cstr = std::ffi::CStr::from_bytes_with_nul(sym_data)
+                    .context("failed to convert to cstr")?;
+                let sym_str = cstr.to_str().context("failed to convert to str")?;
+                let alias = sym_str.replace("alias=", "");
+                writeln!(output, "alias {} {}", alias, module_name).expect("write");
+            }
+            Ok(())
+        })?;
+    Ok(())
+}

test-distro/src/depmod.rs

@@ -57,6 +57,14 @@ fn run() -> anyhow::Result<()> {
             data: None,
             target_mode: Some(RXRXRX),
         },
+        Mount {
+            source: "dev",
+            target: "/dev",
+            fstype: "devtmpfs",
+            flags: nix::mount::MsFlags::empty(),
+            data: None,
+            target_mode: None,
+        },
         Mount {
             source: "sysfs",
             target: "/sys",

init/src/main.rs renamed to test-distro/src/init.rs

@@ -0,0 +1,30 @@
+use std::path::PathBuf;
+
+use anyhow::Context as _;
+use nix::sys::utsname::uname;
+
+/// Kernel modules are in `/lib/modules`.
+/// They may be in the root of this directory,
+/// or in subdirectory named after the kernel release.
+pub fn resolve_modules_dir() -> anyhow::Result<PathBuf> {
+    let modules_dir = PathBuf::from("/lib/modules");
+    let stat = modules_dir
+        .metadata()
+        .with_context(|| format!("{} doesn't exist", modules_dir.display()))?;
+    if stat.is_dir() {
+        return Ok(modules_dir);
+    }
+
+    let utsname = uname().context("failed to get kernel release")?;
+    let release = utsname.release();
+    let modules_dir = modules_dir.join(release);
+    let stat = modules_dir
+        .metadata()
+        .with_context(|| format!("{} doesn't exist", modules_dir.display()))?;
+    anyhow::ensure!(
+        stat.is_dir(),
+        "{} is not a directory",
+        modules_dir.display()
+    );
+    Ok(modules_dir)
+}