Merge pull request #1583 from PrimalPimmy/apparmor-le

feat: lenient apparmor profiles

Author: daemon1024

KubeArmor/core/containerdHandler.go

@@ -7,11 +7,14 @@ package core
 import (
 	"context"
 	"fmt"
+	"github.com/containerd/typeurl/v2"
 	"os"
 	"strconv"
 	"strings"
 	"time"
 
+	"golang.org/x/exp/slices"
+
 	kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
 	cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
 	kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
@@ -21,7 +24,6 @@ import (
 	pb "github.com/containerd/containerd/api/services/containers/v1"
 	pt "github.com/containerd/containerd/api/services/tasks/v1"
 	"github.com/containerd/containerd/namespaces"
-	"github.com/containerd/typeurl/v2"
 	"google.golang.org/grpc"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -31,6 +33,26 @@ import (
 // == Containerd Handler == //
 // ======================== //
 
+// DefaultCaps contains all the default capabilities given to a
+// container by containerd runtime
+// Taken from - https://github.com/containerd/containerd/blob/main/oci/spec.go
+var defaultCaps = []string{
+	"CAP_CHOWN",
+	"CAP_DAC_OVERRIDE",
+	"CAP_FSETID",
+	"CAP_FOWNER",
+	"CAP_MKNOD",
+	"CAP_NET_RAW",
+	"CAP_SETGID",
+	"CAP_SETUID",
+	"CAP_SETFCAP",
+	"CAP_SETPCAP",
+	"CAP_NET_BIND_SERVICE",
+	"CAP_SYS_CHROOT",
+	"CAP_KILL",
+	"CAP_AUDIT_WRITE",
+}
+
 // Containerd Handler
 var Containerd *ContainerdHandler
 
@@ -148,6 +170,11 @@ func (ch *ContainerdHandler) GetContainerInfo(ctx context.Context, containerID s
 	spec := iface.(*specs.Spec)
 	container.AppArmorProfile = spec.Process.ApparmorProfile
 
+	// if a container has additional caps than default, we mark it as privileged
+	if spec.Process.Capabilities != nil && slices.Compare(spec.Process.Capabilities.Permitted, defaultCaps) >= 0 {
+		container.Privileged = true
+	}
+
 	// == //
 
 	taskReq := pt.ListPidsRequest{ContainerID: container.ContainerID}
@@ -305,6 +332,10 @@ func (dm *KubeArmorDaemon) UpdateContainerdContainer(ctx context.Context, contai
 						dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
 					}
 
+					if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
+						dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
+					}
+
 					break
 				}
 			}

KubeArmor/core/crioHandler.go

@@ -119,6 +119,7 @@ func (ch *CrioHandler) GetContainerInfo(ctx context.Context, containerID string)
 
 	// path to container's root storage
 	container.AppArmorProfile = containerInfo.RuntimeSpec.Process.ApparmorProfile
+	container.Privileged = containerInfo.Privileged
 
 	pid := strconv.Itoa(containerInfo.Pid)
 
@@ -240,6 +241,10 @@ func (dm *KubeArmorDaemon) UpdateCrioContainer(ctx context.Context, containerID,
 						dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
 					}
 
+					if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
+						dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
+					}
+
 					break
 				}
 			}

KubeArmor/core/dockerHandler.go

@@ -126,6 +126,9 @@ func (dh *DockerHandler) GetContainerInfo(containerID string) (tp.Container, err
 	}
 
 	container.AppArmorProfile = inspect.AppArmorProfile
+	if inspect.HostConfig != nil {
+		container.Privileged = inspect.HostConfig.Privileged
+	}
 
 	// == //
 
@@ -294,6 +297,10 @@ func (dm *KubeArmorDaemon) GetAlreadyDeployedDockerContainers() {
 								dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
 							}
 
+							if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
+								dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
+							}
+
 							break
 						}
 					}

KubeArmor/core/kubeUpdate.go

@@ -268,6 +268,11 @@ func (dm *KubeArmorDaemon) UpdateEndPointWithPod(action string, pod tp.K8sPod) {
 				newPoint.AppArmorProfiles = append(newPoint.AppArmorProfiles, container.AppArmorProfile)
 			}
 
+			// if container is privileged
+			if _, ok := pod.PrivilegedContainers[container.ContainerName]; ok {
+				container.Privileged = true
+			}
+
 			dm.Containers[containerID] = container
 
 			// in case if container runtime detect the container and emit that event before pod event then
@@ -433,6 +438,11 @@ func (dm *KubeArmorDaemon) UpdateEndPointWithPod(action string, pod tp.K8sPod) {
 					newEndPoint.AppArmorProfiles = append(newEndPoint.AppArmorProfiles, container.AppArmorProfile)
 				}
 
+				// if container is privileged
+				if _, ok := pod.PrivilegedContainers[container.ContainerName]; ok {
+					container.Privileged = true
+				}
+
 				dm.Containers[containerID] = container
 				// in case if container runtime detect the container and emit that event before pod event then
 				// the container id will be added to NsMap with "Unknown" namespace
@@ -688,6 +698,8 @@ func (dm *KubeArmorDaemon) WatchK8sPods() {
 					}
 				}
 
+				pod.PrivilegedContainers = make(map[string]struct{})
+				pod.PrivilegedAppArmorProfiles = make(map[string]struct{})
 				if dm.RuntimeEnforcer != nil && dm.RuntimeEnforcer.EnforcerType == "AppArmor" {
 					appArmorAnnotations := map[string]string{}
 					updateAppArmor := false
@@ -723,7 +735,7 @@ func (dm *KubeArmorDaemon) WatchK8sPods() {
 								}
 							}
 						} else if pod.Metadata["owner.controller"] == "Pod" {
-							pod, err := K8s.K8sClient.CoreV1().Pods("default").Get(context.Background(), "my-pod", metav1.GetOptions{})
+							pod, err := K8s.K8sClient.CoreV1().Pods(pod.Metadata["namespaceName"]).Get(context.Background(), podOwnerName, metav1.GetOptions{})
 							if err == nil {
 								for _, c := range pod.Spec.Containers {
 									containers = append(containers, c.Name)
@@ -747,15 +759,30 @@ func (dm *KubeArmorDaemon) WatchK8sPods() {
 					}
 
 					for _, container := range event.Object.Spec.Containers {
+						var privileged bool
+						// store privileged containers
+						if container.SecurityContext != nil &&
+							((container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged) ||
+								(container.SecurityContext.Capabilities != nil && len(container.SecurityContext.Capabilities.Add) > 0)) {
+							pod.PrivilegedContainers[container.Name] = struct{}{}
+							privileged = true
+						}
+						profileName := "kubearmor-" + pod.Metadata["namespaceName"] + "-" + podOwnerName + "-" + container.Name
 						if _, ok := appArmorAnnotations[container.Name]; !ok && kl.ContainsElement(containers, container.Name) {
-							appArmorAnnotations[container.Name] = "kubearmor-" + pod.Metadata["namespaceName"] + "-" + podOwnerName + "-" + container.Name
+							appArmorAnnotations[container.Name] = profileName
 							updateAppArmor = true
+							// if the container is privileged or it has more than one capabilities added
+							// handle the apparmor profile generation with privileged rules
+						}
+						if privileged {
+							// container name is unique for all containers in a pod
+							pod.PrivilegedAppArmorProfiles[profileName] = struct{}{}
 						}
 					}
 
 					if event.Type == "ADDED" {
 						// update apparmor profiles
-						dm.RuntimeEnforcer.UpdateAppArmorProfiles(pod.Metadata["podName"], "ADDED", appArmorAnnotations)
+						dm.RuntimeEnforcer.UpdateAppArmorProfiles(pod.Metadata["podName"], "ADDED", appArmorAnnotations, pod.PrivilegedAppArmorProfiles)
 
 						if updateAppArmor && pod.Annotations["kubearmor-policy"] == "enabled" {
 							if deploymentName, ok := pod.Metadata["owner.controllerName"]; ok {
@@ -794,7 +821,7 @@ func (dm *KubeArmorDaemon) WatchK8sPods() {
 						}
 					} else if event.Type == "DELETED" {
 						// update apparmor profiles
-						dm.RuntimeEnforcer.UpdateAppArmorProfiles(pod.Metadata["podName"], "DELETED", appArmorAnnotations)
+						dm.RuntimeEnforcer.UpdateAppArmorProfiles(pod.Metadata["podName"], "DELETED", appArmorAnnotations, pod.PrivilegedAppArmorProfiles)
 					}
 				}
 

KubeArmor/core/unorchestratedUpdates.go

@@ -430,6 +430,7 @@ func (dm *KubeArmorDaemon) ParseAndUpdateContainerSecurityPolicy(event tp.K8sKub
 	endPointIndex := -1
 	newPoint := tp.EndPoint{}
 
+	var privilegedProfiles map[string]struct{}
 	for idx, endPoint := range dm.EndPoints {
 		endPointIndex++
 
@@ -509,7 +510,7 @@ func (dm *KubeArmorDaemon) ParseAndUpdateContainerSecurityPolicy(event tp.K8sKub
 	}
 
 	if event.Type == "ADDED" {
-		dm.RuntimeEnforcer.UpdateAppArmorProfiles(containername, "ADDED", appArmorAnnotations)
+		dm.RuntimeEnforcer.UpdateAppArmorProfiles(containername, "ADDED", appArmorAnnotations, privilegedProfiles)
 
 		newPoint.SecurityPolicies = append(newPoint.SecurityPolicies, secPolicy)
 		if i < 0 {
@@ -524,6 +525,9 @@ func (dm *KubeArmorDaemon) ParseAndUpdateContainerSecurityPolicy(event tp.K8sKub
 			newPoint.NetworkVisibilityEnabled = true
 			newPoint.CapabilitiesVisibilityEnabled = true
 			newPoint.Containers = []string{}
+
+			newPoint.PrivilegedContainers = map[string]struct{}{}
+
 			dm.ContainersLock.Lock()
 			for idx, ctr := range dm.Containers {
 				if ctr.ContainerName == containername {

KubeArmor/enforcer/appArmorEnforcer.go

@@ -29,6 +29,8 @@ type AppArmorEnforcer struct {
 
 	// default profile
 	ApparmorDefault string
+	// default privileged profile
+	ApparmorDefaultPrivileged string
 
 	// host profile
 	HostProfile string
@@ -37,6 +39,10 @@ type AppArmorEnforcer struct {
 	AppArmorProfiles     map[string][]string
 	AppArmorProfilesLock *sync.RWMutex
 
+	// to keep track of privileged profiles for clean deletion
+	AppArmorPrivilegedProfiles     map[string]struct{}
+	AppArmorPrivilegedProfilesLock *sync.RWMutex
+
 	// Regex used to get profile Names
 	rgx *regexp.Regexp
 }
@@ -50,38 +56,39 @@ func NewAppArmorEnforcer(node tp.Node, logger *fd.Feeder) *AppArmorEnforcer {
 
 	// default profile
 	ae.ApparmorDefault = `## == Managed by KubeArmor == ##
-	
 #include <tunables/global>
+
 profile apparmor-default flags=(attach_disconnected,mediate_deleted) {
-## == PRE START == ##	
-#include <abstractions/base>	
-umount,
-file,
-network,
-capability,
+## == PRE START == ##
+` + AppArmorDefaultPreStart +
+		`
+## == PRE END == ##
+
+## == POLICY START == ##
+## == POLICY END == ##
+
+## == POST START == ##
+` + AppArmorDefaultPostStart +
+		`
+## == POST END == ##
+}
+`
+
+	ae.ApparmorDefaultPrivileged = `## == Managed by KubeArmor == ##
+#include <tunables/global>
+
+profile apparmor-default flags=(attach_disconnected,mediate_deleted) {
+## == PRE START == ##
+` + AppArmorPrivilegedPreStart +
+		`
 ## == PRE END == ##
 
 ## == POLICY START == ##
 ## == POLICY END == ##
 
 ## == POST START == ##
-/lib/x86_64-linux-gnu/{*,**} rm,
-
-deny @{PROC}/{*,**^[0-9*],sys/kernel/shm*} wkx,
-deny @{PROC}/sysrq-trigger rwklx,
-deny @{PROC}/mem rwklx,
-deny @{PROC}/kmem rwklx,
-deny @{PROC}/kcore rwklx,
-
-deny mount,
-
-deny /sys/[^f]*/** wklx,
-deny /sys/f[^s]*/** wklx,
-deny /sys/fs/[^c]*/** wklx,
-deny /sys/fs/c[^g]*/** wklx,
-deny /sys/fs/cg[^r]*/** wklx,
-deny /sys/firmware/efi/efivars/** rwklx,
-deny /sys/kernel/security/** rwklx,
+` + AppArmorPrivilegedPostStart +
+		`
 ## == POST END == ##
 }
 `
@@ -96,6 +103,9 @@ deny /sys/kernel/security/** rwklx,
 	ae.AppArmorProfiles = map[string][]string{}
 	ae.AppArmorProfilesLock = &sync.RWMutex{}
 
+	ae.AppArmorPrivilegedProfiles = map[string]struct{}{}
+	ae.AppArmorPrivilegedProfilesLock = new(sync.RWMutex)
+
 	files, err := os.ReadDir("/etc/apparmor.d")
 	if err != nil {
 		ae.Logger.Errf("Failed to read /etc/apparmor.d (%s)", err.Error())
@@ -173,7 +183,8 @@ func (ae *AppArmorEnforcer) DestroyAppArmorEnforcer() error {
 	}
 
 	for profile := range ae.AppArmorProfiles {
-		ae.UnregisterAppArmorProfile("", profile)
+		_, privileged := ae.AppArmorPrivilegedProfiles[profile]
+		ae.UnregisterAppArmorProfile("", profile, privileged)
 	}
 
 	if cfg.GlobalCfg.HostPolicy {
@@ -190,7 +201,7 @@ func (ae *AppArmorEnforcer) DestroyAppArmorEnforcer() error {
 // ================================= //
 
 // RegisterAppArmorProfile Function
-func (ae *AppArmorEnforcer) RegisterAppArmorProfile(podName, profileName string) bool {
+func (ae *AppArmorEnforcer) RegisterAppArmorProfile(podName, profileName string, privileged bool) bool {
 	// skip if AppArmorEnforcer is not active
 	if ae == nil {
 		return true
@@ -216,7 +227,15 @@ func (ae *AppArmorEnforcer) RegisterAppArmorProfile(podName, profileName string)
 		return true
 	}
 
-	newProfile := strings.Replace(ae.ApparmorDefault, "apparmor-default", profileName, -1)
+	// generate a profile with basic allows if a privileged container
+	var newProfile string
+	if privileged {
+		newProfile = strings.Replace(ae.ApparmorDefaultPrivileged, "apparmor-default", profileName, -1)
+		ae.AppArmorPrivilegedProfiles[profileName] = struct{}{}
+		ae.Logger.Printf("Added an AppArmor profile for a privileged container (%s, %s)", podName, profileName)
+	} else {
+		newProfile = strings.Replace(ae.ApparmorDefault, "apparmor-default", profileName, -1)
+	}
 
 	newFile, err := os.Create(filepath.Clean("/etc/apparmor.d/" + profileName))
 	if err != nil {
@@ -247,7 +266,7 @@ func (ae *AppArmorEnforcer) RegisterAppArmorProfile(podName, profileName string)
 }
 
 // UnregisterAppArmorProfile Function
-func (ae *AppArmorEnforcer) UnregisterAppArmorProfile(podName, profileName string) bool {
+func (ae *AppArmorEnforcer) UnregisterAppArmorProfile(podName, profileName string, privileged bool) bool {
 	// skip if AppArmorEnforcer is not active
 	if ae == nil {
 		return true
@@ -285,7 +304,12 @@ func (ae *AppArmorEnforcer) UnregisterAppArmorProfile(podName, profileName strin
 		return false
 	}
 
-	newProfile := strings.Replace(ae.ApparmorDefault, "apparmor-default", profileName, -1)
+	var newProfile string
+	if privileged {
+		newProfile = strings.Replace(ae.ApparmorDefaultPrivileged, "apparmor-default", profileName, -1)
+	} else {
+		newProfile = strings.Replace(ae.ApparmorDefault, "apparmor-default", profileName, -1)
+	}
 
 	newFile, err := os.Create(filepath.Clean("/etc/apparmor.d/" + profileName))
 	if err != nil {
@@ -454,7 +478,15 @@ func (ae *AppArmorEnforcer) UnregisterAppArmorHostProfile() bool {
 
 // UpdateAppArmorProfile Function
 func (ae *AppArmorEnforcer) UpdateAppArmorProfile(endPoint tp.EndPoint, appArmorProfile string, securityPolicies []tp.SecurityPolicy) {
-	if policyCount, newProfile, ok := ae.GenerateAppArmorProfile(appArmorProfile, securityPolicies, endPoint.DefaultPosture); ok {
+
+	/* For privileged profiles, we maintain a separate map so that privileged pods
+	   are identified separately and their profiles are generated accordingly
+	*/
+	ae.AppArmorPrivilegedProfilesLock.Lock()
+	_, privileged := ae.AppArmorPrivilegedProfiles[appArmorProfile]
+	ae.AppArmorPrivilegedProfilesLock.Unlock()
+
+	if policyCount, newProfile, ok := ae.GenerateAppArmorProfile(appArmorProfile, securityPolicies, endPoint.DefaultPosture, privileged); ok {
 		newfile, err := os.Create(filepath.Clean("/etc/apparmor.d/" + appArmorProfile))
 		if err != nil {
 			ae.Logger.Warnf("Unable to open an AppArmor profile (%s, %s)", appArmorProfile, err.Error())