ci: let the sandbox canary keep the capabilities its namespace grants

.github/workflows/e2e.yml

@@ -51,6 +51,20 @@ jobs:
         id: setup-bats
         uses: bats-core/bats-action@77d6fb60505b4d0d1d73e48bd035b55074bbfb43 # 4.0.0
 
+      # The sandbox canary builds its own namespaces from inside a pod, and a
+      # user namespace is what grants it CAP_NET_ADMIN for the tun. Ubuntu
+      # 24.04 confines an AppArmor-unconfined process that creates one to the
+      # unprivileged_userns profile, which denies every capability -- so the
+      # namespace is created and the tun is then refused. kind runs its pods
+      # unconfined, so every sandbox in this cluster is that process.
+      - name: Let an unprivileged user namespace keep its capabilities
+        shell: bash
+        run: |
+          restrict=/proc/sys/kernel/apparmor_restrict_unprivileged_userns
+          if [[ -e "${restrict}" ]]; then
+            sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+          fi
+
       - name: Bats tests
         shell: bash
         env:

cmd/nano-init/isolation.go

@@ -80,10 +80,26 @@ func isolationError(links []string) error {
 // is the whole reason for the diagnosis below.
 const tunDevice = "/dev/net/tun"
 
+// userNSRestriction is Ubuntu's switch for what an unprivileged user namespace
+// may do.
+const userNSRestriction = "/proc/sys/kernel/apparmor_restrict_unprivileged_userns"
+
+// userNSCapabilitiesRestricted reports whether this host takes back the
+// capabilities a user namespace would otherwise grant.
+//
+// Where it is on, a process with no AppArmor profile that creates a user
+// namespace is confined to the unprivileged_userns profile, and that profile
+// denies every capability. Nothing fails at the time: the namespace is created,
+// and the capability inside it is refused later.
+func userNSCapabilitiesRestricted() bool {
+	value, err := os.ReadFile(userNSRestriction)
+	return err == nil && strings.TrimSpace(string(value)) != "0"
+}
+
 // describeTunFailure turns a netlink error into the thing to change.
 func describeTunFailure(err error) string {
 	_, statErr := os.Stat(tunDevice)
-	return tunHint(err, statErr)
+	return tunHint(err, statErr, userNSCapabilitiesRestricted())
 }
 
 // tunHint explains a failed tun creation.
@@ -93,7 +109,7 @@ func describeTunFailure(err error) string {
 // means the device was not passed in or the capability was not granted. The
 // profiles fail differently, so they are told apart here rather than left to
 // whoever is reading a log at the time.
-func tunHint(err error, statErr error) string {
+func tunHint(err error, statErr error, restrictedUserNS bool) string {
 	switch {
 	case os.IsNotExist(statErr):
 		return "there is no " + tunDevice + ". In a microVM that means a guest kernel built without CONFIG_TUN" +
@@ -110,6 +126,16 @@ func tunHint(err error, statErr error) string {
 		// netlink formats this one rather than wrapping the errno, so there is
 		// nothing for errors.Is to match on.
 		err != nil && strings.Contains(err.Error(), "TUNSETIFF"):
+		// Named first because the advice below is wrong here: the capability
+		// was granted and then taken away again, so granting it harder is no
+		// answer.
+		if restrictedUserNS {
+			return "creating a tun was refused, and this host restricts what an unprivileged user namespace may do" +
+				" (" + userNSRestriction + " is not 0): a process with no AppArmor profile that creates one is" +
+				" confined to the unprivileged_userns profile, which denies every capability -- so the namespace" +
+				" was created and CAP_NET_ADMIN in it was refused anyway. Give the sandbox an AppArmor profile" +
+				" that permits capabilities, or set that sysctl to 0"
+		}
 		return "creating a tun was refused. It needs CAP_NET_ADMIN in the user namespace that owns this network" +
 			" namespace: a user namespace of your own grants it over the namespaces it owns, which is what" +
 			" --create-namespaces relies on; otherwise `--cap-add NET_ADMIN` for docker, or" +

cmd/nano-init/isolation_test.go

@@ -30,6 +30,9 @@ func TestTunHint(t *testing.T) {
 		name    string
 		err     error
 		statErr error
+		// restricted is a host that hands back a user namespace and then
+		// denies the capabilities it would have granted.
+		restricted bool
 		// want is a phrase naming the fix, so the test fails when the advice
 		// stops matching the cause rather than only when the wording changes.
 		want string
@@ -63,6 +66,21 @@ func TestTunHint(t *testing.T) {
 			err:  syscall.EPERM,
 			want: "CAP_NET_ADMIN",
 		},
+		{
+			// The capability advice is worse than useless here: it was granted
+			// by the namespace and taken back by the host.
+			name:       "a host that strips a user namespace's capabilities names the sysctl",
+			err:        syscall.EPERM,
+			restricted: true,
+			want:       "apparmor_restrict_unprivileged_userns",
+		},
+		{
+			name:       "a device that cannot be opened is still not blamed on the sysctl",
+			err:        syscall.EPERM,
+			statErr:    denied,
+			restricted: true,
+			want:       "cannot be opened",
+		},
 		{
 			name: "EACCES is treated as EPERM is",
 			err:  syscall.EACCES,
@@ -80,9 +98,9 @@ func TestTunHint(t *testing.T) {
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			got := tunHint(tc.err, tc.statErr)
+			got := tunHint(tc.err, tc.statErr, tc.restricted)
 			if !strings.Contains(got, tc.want) {
-				t.Errorf("tunHint(%v, %v) = %q, want it to mention %q", tc.err, tc.statErr, got, tc.want)
+				t.Errorf("tunHint(%v, %v, %v) = %q, want it to mention %q", tc.err, tc.statErr, tc.restricted, got, tc.want)
 			}
 		})
 	}