From fde9b832b34b3571a5a5e38630e6fe36eeb57471 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 25 Aug 2026 13:30:05 +0000 Subject: [PATCH] ci: let the sandbox canary keep the capabilities its namespace grants The bats suite moved from ubuntu-22.04-16core to ubuntu-latest, and the canary stopped being a sandbox: create tun0: Tuntap IOCTL TUNSETIFF failed [0], errno operation not permitted Ubuntu 24.04 sets kernel.apparmor_restrict_unprivileged_userns=1. Under it, a process with no AppArmor profile that creates a user namespace is confined to the unprivileged_userns profile, and that profile denies every capability -- so --create-namespaces gets its namespace and is then refused the CAP_NET_ADMIN the namespace exists to grant. kind runs its pods unconfined, so every sandbox in that cluster is that process. Only this suite is affected. tests/integration drives the same path on the same runner and passes, because it runs under `sudo make test` and the restriction mediates unprivileged userns creation only. agent_sandbox.bats grants --cap-add NET_ADMIN and never takes the namespace route at all. The sysctl is cleared on the runner rather than the manifest being given a securityContext: GKE, where this canary deploys, applies an AppArmor profile to its pods, so the transition never happens there, and the manifest under test should stay the manifest that ships. nano-init's own diagnosis was worse than useless here -- it asked for CAP_NET_ADMIN, which the namespace had granted and the host had taken back -- so it now reads the sysctl and names it. That is the only thing a contributor running the suite on their own 24.04 workstation has to go on. --- .github/workflows/e2e.yml | 14 ++++++++++++++ cmd/nano-init/isolation.go | 30 ++++++++++++++++++++++++++++-- cmd/nano-init/isolation_test.go | 22 ++++++++++++++++++++-- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9891407d..8280be03 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,6 +51,20 @@ jobs: id: setup-bats uses: bats-core/bats-action@77d6fb60505b4d0d1d73e48bd035b55074bbfb43 # 4.0.0 + # The sandbox canary builds its own namespaces from inside a pod, and a + # user namespace is what grants it CAP_NET_ADMIN for the tun. Ubuntu + # 24.04 confines an AppArmor-unconfined process that creates one to the + # unprivileged_userns profile, which denies every capability -- so the + # namespace is created and the tun is then refused. kind runs its pods + # unconfined, so every sandbox in this cluster is that process. + - name: Let an unprivileged user namespace keep its capabilities + shell: bash + run: | + restrict=/proc/sys/kernel/apparmor_restrict_unprivileged_userns + if [[ -e "${restrict}" ]]; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + - name: Bats tests shell: bash env: diff --git a/cmd/nano-init/isolation.go b/cmd/nano-init/isolation.go index 81da3c5a..930593be 100644 --- a/cmd/nano-init/isolation.go +++ b/cmd/nano-init/isolation.go @@ -80,10 +80,26 @@ func isolationError(links []string) error { // is the whole reason for the diagnosis below. const tunDevice = "/dev/net/tun" +// userNSRestriction is Ubuntu's switch for what an unprivileged user namespace +// may do. +const userNSRestriction = "/proc/sys/kernel/apparmor_restrict_unprivileged_userns" + +// userNSCapabilitiesRestricted reports whether this host takes back the +// capabilities a user namespace would otherwise grant. +// +// Where it is on, a process with no AppArmor profile that creates a user +// namespace is confined to the unprivileged_userns profile, and that profile +// denies every capability. Nothing fails at the time: the namespace is created, +// and the capability inside it is refused later. +func userNSCapabilitiesRestricted() bool { + value, err := os.ReadFile(userNSRestriction) + return err == nil && strings.TrimSpace(string(value)) != "0" +} + // describeTunFailure turns a netlink error into the thing to change. func describeTunFailure(err error) string { _, statErr := os.Stat(tunDevice) - return tunHint(err, statErr) + return tunHint(err, statErr, userNSCapabilitiesRestricted()) } // tunHint explains a failed tun creation. @@ -93,7 +109,7 @@ func describeTunFailure(err error) string { // means the device was not passed in or the capability was not granted. The // profiles fail differently, so they are told apart here rather than left to // whoever is reading a log at the time. -func tunHint(err error, statErr error) string { +func tunHint(err error, statErr error, restrictedUserNS bool) string { switch { case os.IsNotExist(statErr): return "there is no " + tunDevice + ". In a microVM that means a guest kernel built without CONFIG_TUN" + @@ -110,6 +126,16 @@ func tunHint(err error, statErr error) string { // netlink formats this one rather than wrapping the errno, so there is // nothing for errors.Is to match on. err != nil && strings.Contains(err.Error(), "TUNSETIFF"): + // Named first because the advice below is wrong here: the capability + // was granted and then taken away again, so granting it harder is no + // answer. + if restrictedUserNS { + return "creating a tun was refused, and this host restricts what an unprivileged user namespace may do" + + " (" + userNSRestriction + " is not 0): a process with no AppArmor profile that creates one is" + + " confined to the unprivileged_userns profile, which denies every capability -- so the namespace" + + " was created and CAP_NET_ADMIN in it was refused anyway. Give the sandbox an AppArmor profile" + + " that permits capabilities, or set that sysctl to 0" + } return "creating a tun was refused. It needs CAP_NET_ADMIN in the user namespace that owns this network" + " namespace: a user namespace of your own grants it over the namespaces it owns, which is what" + " --create-namespaces relies on; otherwise `--cap-add NET_ADMIN` for docker, or" + diff --git a/cmd/nano-init/isolation_test.go b/cmd/nano-init/isolation_test.go index 38257b75..2223e363 100644 --- a/cmd/nano-init/isolation_test.go +++ b/cmd/nano-init/isolation_test.go @@ -30,6 +30,9 @@ func TestTunHint(t *testing.T) { name string err error statErr error + // restricted is a host that hands back a user namespace and then + // denies the capabilities it would have granted. + restricted bool // want is a phrase naming the fix, so the test fails when the advice // stops matching the cause rather than only when the wording changes. want string @@ -63,6 +66,21 @@ func TestTunHint(t *testing.T) { err: syscall.EPERM, want: "CAP_NET_ADMIN", }, + { + // The capability advice is worse than useless here: it was granted + // by the namespace and taken back by the host. + name: "a host that strips a user namespace's capabilities names the sysctl", + err: syscall.EPERM, + restricted: true, + want: "apparmor_restrict_unprivileged_userns", + }, + { + name: "a device that cannot be opened is still not blamed on the sysctl", + err: syscall.EPERM, + statErr: denied, + restricted: true, + want: "cannot be opened", + }, { name: "EACCES is treated as EPERM is", err: syscall.EACCES, @@ -80,9 +98,9 @@ func TestTunHint(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - got := tunHint(tc.err, tc.statErr) + got := tunHint(tc.err, tc.statErr, tc.restricted) if !strings.Contains(got, tc.want) { - t.Errorf("tunHint(%v, %v) = %q, want it to mention %q", tc.err, tc.statErr, got, tc.want) + t.Errorf("tunHint(%v, %v, %v) = %q, want it to mention %q", tc.err, tc.statErr, tc.restricted, got, tc.want) } }) }