feat: Refactor init into test-distro

.github/scripts/find_kernels.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+import os
+import glob
+import sys
+from typing import List
+
+def find_kernels(directory: str) -> List[str]:
+    return glob.glob(f"{directory}/**/vmlinuz-*", recursive=True)
+
+def find_modules_directory(directory: str, kernel: str) -> str:
+    matches = glob.glob(f"{directory}/**/modules/{kernel}", recursive=True)
+    if len(matches) != 1:
+        raise RuntimeError(f"Expected to find exactly one modules directory. Found {len(matches)}.")
+    return matches[0]
+
+def main() -> None:
+    images = find_kernels('test/.tmp')
+    modules = []
+
+    for image in images:
+        image_name = os.path.basename(image).replace('vmlinuz-', '')
+        module_dir = find_modules_directory('test/.tmp', image_name)
+        modules.append(module_dir)
+
+    args = ' '.join(f"{image}:{module}" for image, module in zip(images, modules))
+    print(args)
+
+if __name__ == "__main__":
+    main()

.github/workflows/ci.yml

@@ -228,7 +228,7 @@ jobs:
         run: |
           set -euxo pipefail
           sudo apt update
-          sudo apt -y install lynx qemu-system-{arm,x86}
+          sudo apt -y install lynx qemu-system-{arm,x86} musl-tools
           echo /usr/lib/llvm-15/bin >> $GITHUB_PATH
 
       - name: Install prerequisites
@@ -240,6 +240,10 @@ jobs:
         # The tar shipped on macOS doesn't support --wildcards, so we need GNU tar.
         #
         # The clang shipped on macOS doesn't support BPF, so we need LLVM from brew.
+        #
+        # We need a musl C toolchain to compile our `test-distro` since some of
+        # our dependencies have build scripts that compile C code (i.e xz2).
+        # This is provided by `brew install filosottile/musl-cross/musl-cross`.
         run: |
           set -euxo pipefail
           brew update
@@ -250,6 +254,8 @@ jobs:
           echo $(brew --prefix curl)/bin >> $GITHUB_PATH
           echo $(brew --prefix gnu-tar)/libexec/gnubin >> $GITHUB_PATH
           echo $(brew --prefix llvm)/bin >> $GITHUB_PATH
+          brew install filosottile/musl-cross/musl-cross
+          ln -s "$(brew --prefix musl-cross)/bin/x86_64-linux-musl-gcc" /usr/local/bin/musl-gcc
 
       - uses: dtolnay/rust-toolchain@nightly
         with:
@@ -302,21 +308,46 @@ jobs:
         # TODO: enable tests on kernels before 6.0.
         run: .github/scripts/download_kernel_images.sh test/.tmp/debian-kernels/amd64 amd64 6.1 6.10
 
+      - name: Cleanup stale kernels and modules
+        run: |
+          set -euxo pipefail
+          rm -rf test/.tmp/boot test/.tmp/lib
+
       - name: Extract debian kernels
         run: |
           set -euxo pipefail
+          # The wildcard '**/boot/*' extracts kernel images and config.
+          # The wildcard '**/modules/*' extracts kernel modules.
+          # Modules are required since not all parts of the kernel we want to
+          # test are built-in.
           find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \
-            sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -"
+            sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp \
+              --wildcards --extract '**/boot/*' '**/modules/*' --file -"
 
       - name: Run local integration tests
         if: runner.os == 'Linux'
         run: cargo xtask integration-test local
 
       - name: Run virtualized integration tests
+        if: runner.os == 'Linux'
+        run: |
+          set -euxo pipefail
+          ARGS=$(./.github/scripts/find_kernels.py)
+          cargo xtask integration-test vm --cache-dir test/.tmp \
+            --github-api-token ${{ secrets.GITHUB_TOKEN }} \
+            ${ARGS}
+
+      - name: Run virtualized integration tests
+        if: runner.os == 'macOS'
+        env:
+          # This sets the linker to the one installed by FiloSottile/musl-cross.
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: x86_64-linux-musl-gcc
         run: |
           set -euxo pipefail
-          find test/.tmp -name 'vmlinuz-*' -print0 | xargs -t -0 \
-            cargo xtask integration-test vm --cache-dir test/.tmp --github-api-token ${{ secrets.GITHUB_TOKEN }}
+          ARGS=$(./.github/scripts/find_kernels.py)
+          cargo xtask integration-test vm --cache-dir test/.tmp \
+            --github-api-token ${{ secrets.GITHUB_TOKEN }} \
+            ${ARGS}
 
   # Provides a single status check for the entire build workflow.
   # This is used for merge automation, like Mergify, since GH actions

.vscode/settings.json

@@ -1,7 +1,8 @@
 {
-    "rust-analyzer.check.allTargets": true,
-    "rust-analyzer.check.command": "clippy",
-    "search.exclude": {
-        "/xtask/public-api/*.txt": true,
-    },
+  "rust-analyzer.check.allTargets": true,
+  "rust-analyzer.check.command": "clippy",
+  "search.exclude": {
+    "/xtask/public-api/*.txt": true
+  },
+  "yaml.format.singleQuote": true
 }

Cargo.toml

@@ -7,7 +7,7 @@ members = [
     "aya-log-parser",
     "aya-obj",
     "aya-tool",
-    "init",
+    "test-distro",
     "test/integration-common",
     "test/integration-test",
     "xtask",
@@ -33,7 +33,7 @@ default-members = [
     "aya-log-parser",
     "aya-obj",
     "aya-tool",
-    "init",
+    "test-distro",
     "test/integration-common",
     # test/integration-test is omitted; including it in this list causes `cargo test` to run its
     # tests, and that doesn't work unless they've been built with `cargo xtask`.
@@ -74,6 +74,7 @@ diff = { version = "0.1.13", default-features = false }
 env_logger = { version = "0.11", default-features = false }
 epoll = { version = "4.3.3", default-features = false }
 futures = { version = "0.3.28", default-features = false }
+glob = { version = "0.3.0", default-features = false }
 hashbrown = { version = "0.15.0", default-features = false }
 indoc = { version = "2.0", default-features = false }
 libc = { version = "0.2.105", default-features = false }
@@ -101,8 +102,10 @@ test-log = { version = "0.2.13", default-features = false }
 testing_logger = { version = "0.1.1", default-features = false }
 thiserror = { version = "2.0.3", default-features = false }
 tokio = { version = "1.24.0", default-features = false }
+walkdir = { version = "2", default-features = false }
 which = { version = "7.0.0", default-features = false }
 xdpilone = { version = "1.0.5", default-features = false }
+xz2 = { version = "0.1.7", default-features = false }
 
 [workspace.lints.rust]
 unused-extern-crates = "warn"

test-distro/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "test-distro"
+publish = false
+version = "0.1.0"
+
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[[bin]]
+name = "init"
+path = "src/init.rs"
+
+[[bin]]
+name = "modprobe"
+path = "src/modprobe.rs"
+
+[[bin]]
+name = "depmod"
+path = "src/depmod.rs"
+
+[dependencies]
+anyhow = { workspace = true, features = ["std"] }
+clap = { workspace = true, default-features = true, features = ["derive"] }
+glob = { workspace = true }
+nix = { workspace = true, features = [
+    "user",
+    "fs",
+    "mount",
+    "reboot",
+    "kmod",
+    "feature",
+] }
+object = { workspace = true, features = ["elf", "read_core", "std"] }
+walkdir = { workspace = true }
+xz2 = { workspace = true }

test-distro/src/depmod.rs

@@ -0,0 +1,140 @@
+//! depmod is used to build the modules.alias file to assist with loading
+//! kernel modules.
+//!
+//! This implementation is incredibly naive and is only designed to work within
+//! the constraints of the test environment. Not for production use.
+
+use std::{
+    fs::File,
+    io::{BufWriter, Read, Write as _},
+    path::PathBuf,
+};
+
+use anyhow::{Context as _, anyhow};
+use clap::Parser;
+use object::{Object, ObjectSection, ObjectSymbol, Section};
+use test_distro::resolve_modules_dir;
+use walkdir::WalkDir;
+use xz2::read::XzDecoder;
+
+#[derive(Parser)]
+struct Args {
+    #[clap(long, short)]
+    base_dir: Option<PathBuf>,
+}
+
+fn main() -> anyhow::Result<()> {
+    let Args { base_dir } = Parser::parse();
+
+    let modules_dir = if let Some(base_dir) = base_dir {
+        base_dir
+    } else {
+        resolve_modules_dir().context("failed to resolve modules dir")?
+    };
+
+    let modules_alias = modules_dir.join("modules.alias");
+    let f = std::fs::OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open(&modules_alias)
+        .with_context(|| format!("failed to open: {}", modules_alias.display()))?;
+    let mut output = BufWriter::new(&f);
+    for entry in WalkDir::new(modules_dir) {
+        let entry = entry.context("failed to read entry in walkdir")?;
+        if entry.file_type().is_file() {
+            let path = entry.path();
+
+            let module_name = path
+                .file_name()
+                .ok_or_else(|| anyhow!("{} does not have a file name", path.display()))?
+                .to_str()
+                .ok_or_else(|| anyhow!("{} is not valid utf-8", path.display()))?;
+
+            let (module_name, compressed) =
+                if let Some(module_name) = module_name.strip_suffix(".xz") {
+                    (module_name, true)
+                } else {
+                    (module_name, false)
+                };
+
+            let module_name = if let Some(module_name) = module_name.strip_suffix(".ko") {
+                module_name
+            } else {
+                // Not a kernel module
+                continue;
+            };
+
+            let mut f =
+                File::open(path).with_context(|| format!("failed to open: {}", path.display()))?;
+            let stat = f
+                .metadata()
+                .with_context(|| format!("failed to get metadata for {}", path.display()))?;
+
+            if compressed {
+                let mut decoder = XzDecoder::new(f);
+                // We don't know the size of the decompressed data, so we assume it's
+                // no more than twice the size of the compressed data.
+                let mut decompressed = Vec::with_capacity(stat.len() as usize * 2);
+                decoder.read_to_end(&mut decompressed)?;
+                read_aliases_from_module(&decompressed, module_name, &mut output)
+            } else {
+                let mut buf = Vec::with_capacity(stat.len() as usize);
+                f.read_to_end(&mut buf)
+                    .with_context(|| format!("failed to read: {}", path.display()))?;
+                read_aliases_from_module(&buf, module_name, &mut output)
+            }
+            .with_context(|| format!("failed to read aliases from module {}", path.display()))?;
+        }
+    }
+    Ok(())
+}
+
+fn read_aliases_from_module(
+    contents: &[u8],
+    module_name: &str,
+    output: &mut BufWriter<&File>,
+) -> Result<(), anyhow::Error> {
+    let obj = object::read::File::parse(contents).context("failed to parse")?;
+
+    let section = (|| -> anyhow::Result<Option<Section<'_, '_, &[u8]>>> {
+        for s in obj.sections() {
+            let name = s
+                .name_bytes()
+                .with_context(|| format!("failed to get name of section idx {}", s.index()))?;
+            if name == b".modinfo" {
+                return Ok(Some(s));
+            }
+        }
+        Ok(None)
+    })()?;
+    let section = section.context("failed to find .modinfo section")?;
+    let section_idx = section.index();
+    let data = section
+        .data()
+        .context("failed to get modinfo section data")?;
+
+    for s in obj.symbols() {
+        if s.section_index() != Some(section_idx) {
+            continue;
+        }
+        let name = s
+            .name()
+            .with_context(|| format!("failed to get name of symbol idx {}", s.index()))?;
+        if name.contains("alias") {
+            let start = s.address() as usize;
+            let end = start + s.size() as usize;
+            let sym_data = &data[start..end];
+            let cstr = std::ffi::CStr::from_bytes_with_nul(sym_data)
+                .with_context(|| format!("failed to convert {:?} to cstr", sym_data))?;
+            let sym_str = cstr
+                .to_str()
+                .with_context(|| format!("failed to convert {:?} to str", cstr))?;
+            let alias = sym_str
+                .strip_prefix("alias=")
+                .with_context(|| format!("failed to strip prefix 'alias=' from {}", sym_str))?;
+            writeln!(output, "alias {} {}", alias, module_name).expect("write");
+        }
+    }
+    Ok(())
+}

init/src/main.rs → test-distro/src/init.rs

@@ -57,6 +57,14 @@ fn run() -> anyhow::Result<()> {
             data: None,
             target_mode: Some(RXRXRX),
         },
+        Mount {
+            source: "dev",
+            target: "/dev",
+            fstype: "devtmpfs",
+            flags: nix::mount::MsFlags::empty(),
+            data: None,
+            target_mode: None,
+        },
         Mount {
             source: "sysfs",
             target: "/sys",