From 90af9ec63d8750da4cb47fe2327420e70a8c289e Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Thu, 21 Nov 2024 09:00:02 +0100 Subject: [PATCH] read SocketPath from node config, unset RuntimeProtocol Signed-off-by: Matthias Bertschy --- .../v1/container_watcher_private.go | 6 +- pkg/utils/utils.go | 132 +++++++++--------- .../templates/node-agent/clusterrole.yaml | 4 +- 3 files changed, 75 insertions(+), 67 deletions(-) diff --git a/pkg/containerwatcher/v1/container_watcher_private.go b/pkg/containerwatcher/v1/container_watcher_private.go index 55e6451d..6f641b98 100644 --- a/pkg/containerwatcher/v1/container_watcher_private.go +++ b/pkg/containerwatcher/v1/container_watcher_private.go @@ -36,7 +36,11 @@ func (ch *IGContainerWatcher) containerCallback(notif containercollection.PubSub switch notif.Type { case containercollection.EventTypeAddContainer: - logger.L().Info("start monitor on container", helpers.String("container ID", notif.Container.Runtime.ContainerID), helpers.String("k8s workload", k8sContainerID)) + logger.L().Info("start monitor on container", + helpers.String("container ID", notif.Container.Runtime.ContainerID), + helpers.String("k8s workload", k8sContainerID), + helpers.String("ContainerImageDigest", notif.Container.Runtime.ContainerImageDigest), + helpers.String("ContainerImageName", notif.Container.Runtime.ContainerImageName)) if ch.running { ch.timeBasedContainers.Add(notif.Container.Runtime.ContainerID) } else { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index e78f99f8..ff315d8b 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -6,6 +6,7 @@ import ( "crypto/sha1" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "hash" @@ -17,15 +18,17 @@ import ( "slices" "strconv" "strings" - "syscall" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - mapset "github.com/deckarep/golang-set/v2" - + apitypes "github.com/armosec/armoapi-go/armotypes" "github.com/armosec/utils-k8s-go/wlid" + mapset "github.com/deckarep/golang-set/v2" "github.com/goradd/maps" + runtimeclient "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/runtime-client" + containerutilsTypes "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/types" + tracerexectype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/exec/types" + traceropentype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/open/types" + igtypes "github.com/inspektor-gadget/inspektor-gadget/pkg/types" "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" "github.com/kubescape/k8s-interface/instanceidhandler" @@ -34,18 +37,11 @@ import ( "github.com/kubescape/k8s-interface/k8sinterface" "github.com/kubescape/k8s-interface/workloadinterface" "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/prometheus/procfs" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" - - "github.com/prometheus/procfs" - - runtimeclient "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/runtime-client" - containerutilsTypes "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/types" - tracerexectype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/exec/types" - traceropentype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/open/types" - igtypes "github.com/inspektor-gadget/inspektor-gadget/pkg/types" - - apitypes "github.com/armosec/armoapi-go/armotypes" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" ) var ( @@ -475,9 +471,9 @@ func GetFileSize(path string) (int64, error) { } func CalculateSHA256FileExecHash(path string, args []string) string { - hash := sha256.New() - hash.Write([]byte(fmt.Sprintf("%s;%v", path, args))) - hashInBytes := hash.Sum(nil) + hsh := sha256.New() + hsh.Write([]byte(fmt.Sprintf("%s;%v", path, args))) + hashInBytes := hsh.Sum(nil) return hex.EncodeToString(hashInBytes) } @@ -683,49 +679,6 @@ func ChunkBy[T any](items []T, chunkSize int) [][]T { } // isUnixSocket checks if the given path is a Unix socket. -func isUnixSocket(path string) (bool, error) { - fileInfo, err := os.Stat(path) - if err != nil { - return false, err // Could not obtain the file stats - } - - stat, ok := fileInfo.Sys().(*syscall.Stat_t) - if !ok { - return false, fmt.Errorf("not a unix file") - } - - // Check if the file is a socket - return (stat.Mode & syscall.S_IFMT) == syscall.S_IFSOCK, nil -} - -func DetectContainerRuntimes(hostMount string) ([]*containerutilsTypes.RuntimeConfig, error) { - runtimes := map[igtypes.RuntimeName]string{ - igtypes.RuntimeNameDocker: runtimeclient.DockerDefaultSocketPath, - igtypes.RuntimeNameCrio: runtimeclient.CrioDefaultSocketPath, - igtypes.RuntimeNameContainerd: runtimeclient.ContainerdDefaultSocketPath, - igtypes.RuntimeNamePodman: runtimeclient.PodmanDefaultSocketPath, - } - - detectedRuntimes := make([]*containerutilsTypes.RuntimeConfig, 0) - - for runtimeName, socketPath := range runtimes { - // Check if the socket is available on the host mount - socketPath = hostMount + socketPath - if isSocket, err := isUnixSocket(socketPath); err == nil && isSocket { - logger.L().Info("Detected container runtime", helpers.String("runtime", runtimeName.String()), helpers.String("socketPath", socketPath)) - detectedRuntimes = append(detectedRuntimes, &containerutilsTypes.RuntimeConfig{ - Name: runtimeName, - SocketPath: socketPath, - }) - } - } - - if len(detectedRuntimes) == 0 { - return nil, fmt.Errorf("no container runtimes detected at the following paths: %v", runtimes) - } - - return detectedRuntimes, nil -} func DetectContainerRuntimeViaK8sAPI(ctx context.Context, k8sClient *k8sinterface.KubernetesApi, nodeName string) (*containerutilsTypes.RuntimeConfig, error) { // Get the current node @@ -735,17 +688,23 @@ func DetectContainerRuntimeViaK8sAPI(ctx context.Context, k8sClient *k8sinterfac if err != nil { return nil, fmt.Errorf("failed to list nodes: %v", err) } - if len(nodes.Items) == 0 { return nil, fmt.Errorf("no node found with name: %s", nodeName) } - node := nodes.Items[0] + // parse the runtime info runtimeConfig := parseRuntimeInfo(node.Status.NodeInfo.ContainerRuntimeVersion) if runtimeConfig.Name == igtypes.RuntimeNameUnknown { return nil, fmt.Errorf("unknown container runtime: %s", node.Status.NodeInfo.ContainerRuntimeVersion) } - + // override the socket path + realSocketPath, err := getContainerRuntimeSocketPath(k8sClient, nodeName) + if err != nil { + return nil, fmt.Errorf("failed to get container runtime socket path from Kubelet configz: %v", err) + } + runtimeConfig.SocketPath = realSocketPath + // unset the runtime protocol + runtimeConfig.RuntimeProtocol = "" return runtimeConfig, nil } @@ -783,3 +742,48 @@ func parseRuntimeInfo(version string) *containerutilsTypes.RuntimeConfig { } } } + +func getContainerRuntimeSocketPath(clientset *k8sinterface.KubernetesApi, nodeName string) (string, error) { + kubeletConfig, err := getCurrentKubeletConfig(clientset, nodeName) + if err != nil { + return "", fmt.Errorf("getting /configz: %w", err) + } + socketPath, found := strings.CutPrefix(kubeletConfig.ContainerRuntimeEndpoint, "unix://") + if !found { + return "", fmt.Errorf("socket path does not start with unix://") + } + logger.L().Info("using the detected container runtime socket path from Kubelet's config", helpers.String("socketPath", socketPath)) + return socketPath, nil +} + +// The /configz endpoint isn't officially documented. It was introduced in Kubernetes 1.26 and been around for a long time +// as stated in https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/component-base/configz/OWNERS +func getCurrentKubeletConfig(clientset *k8sinterface.KubernetesApi, nodeName string) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + resp, err := clientset.GetKubernetesClient().CoreV1().RESTClient().Get().Resource("nodes"). + Name(nodeName).Suffix("proxy", "configz").DoRaw(context.TODO()) + if err != nil { + return nil, fmt.Errorf("fetching /configz from %q: %w", nodeName, err) + } + kubeCfg, err := decodeConfigz(resp) + if err != nil { + return nil, err + } + return kubeCfg, nil +} + +// Decodes the http response from /configz and returns the kubelet configuration +func decodeConfigz(respBody []byte) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + // This hack because /configz reports the following structure: + // {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}} + type configzWrapper struct { + ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` + } + + configz := configzWrapper{} + err := json.Unmarshal(respBody, &configz) + if err != nil { + return nil, err + } + + return &configz.ComponentConfig, nil +} diff --git a/tests/chart/templates/node-agent/clusterrole.yaml b/tests/chart/templates/node-agent/clusterrole.yaml index 480a37f1..67f0a98a 100644 --- a/tests/chart/templates/node-agent/clusterrole.yaml +++ b/tests/chart/templates/node-agent/clusterrole.yaml @@ -6,7 +6,7 @@ metadata: kubescape.io/ignore: "true" rules: - apiGroups: [""] - resources: ["pods", "nodes", "services", "endpoints", "namespaces"] + resources: ["pods", "nodes", "nodes/proxy", "services", "endpoints", "namespaces"] verbs: ["get", "watch", "list"] - apiGroups: [""] resources: ["events"] @@ -25,4 +25,4 @@ rules: verbs: ["create", "get", "update", "watch", "list", "patch"] - apiGroups: ["kubescape.io"] resources: ["runtimerulealertbindings"] - verbs: ["list", "watch"] \ No newline at end of file + verbs: ["list", "watch"]