From 89f97b2fc20c70341a5b33dbfa08a7a8f99708b8 Mon Sep 17 00:00:00 2001 From: mrproliu <741550557@qq.com> Date: Wed, 25 Sep 2024 14:49:44 +0800 Subject: [PATCH] Reduce file mount needs when deploy in the Kubernetes --- CHANGES.md | 1 + pkg/accesslog/common/connection.go | 2 +- pkg/process/finders/base/tool.go | 2 +- pkg/process/finders/kubernetes/finder.go | 2 +- pkg/tools/btf/check.go | 58 ------------------- pkg/tools/host/file.go | 34 +++++++---- pkg/tools/operator/distribution.go | 8 +-- pkg/tools/process/process.go | 8 +-- pkg/tools/profiling/api.go | 2 +- pkg/tools/profiling/kernel.go | 4 +- pkg/tools/ssl/gotls.go | 2 +- test/e2e/cases/access_log/rover.yaml | 12 ++-- test/e2e/cases/process/istio/rover.yaml | 12 ++-- .../e2e/cases/profiling/continuous/rover.yaml | 12 ++-- .../cases/profiling/task/network/rover.yaml | 12 ++-- .../cases/profiling/task/offcpu/rover.yaml | 12 ++-- .../e2e/cases/profiling/task/oncpu/rover.yaml | 12 ++-- 17 files changed, 76 insertions(+), 119 deletions(-) delete mode 100644 pkg/tools/btf/check.go diff --git a/CHANGES.md b/CHANGES.md index ad124498..91bf302f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,7 @@ Release Notes. * Introduce `MonitorFilter` into access log module. * Support monitoring ztunnel to adapt istio ambient mode. * Enhance get connection address strategy in access log module. +* Reduce file mount needs when deploy in the Kubernetes, split env name `ROVER_HOST_MAPPING` to `ROVER_HOST_PROC_MAPPING` and `ROVER_HOST_ETC_MAPPING`. #### Bug Fixes * Fixed the issue where `conntrack` could not find the Reply IP in the access log module. diff --git a/pkg/accesslog/common/connection.go b/pkg/accesslog/common/connection.go index fe1065c6..690d7812 100644 --- a/pkg/accesslog/common/connection.go +++ b/pkg/accesslog/common/connection.go @@ -216,7 +216,7 @@ func (c *ConnectionManager) Start(ctx context.Context, accessLogContext *AccessL } func (c *ConnectionManager) checkProcessFDExist(pid, fd uint32) bool { - return path.Exists(host.GetFileInHost(fmt.Sprintf("/proc/%d/fd/%d", pid, fd))) + return path.Exists(host.GetHostProcInHost(fmt.Sprintf("%d/fd/%d", pid, fd))) } func (c *ConnectionManager) Stop() { diff --git a/pkg/process/finders/base/tool.go b/pkg/process/finders/base/tool.go index 749d4f68..f2413be1 100644 --- a/pkg/process/finders/base/tool.go +++ b/pkg/process/finders/base/tool.go @@ -52,7 +52,7 @@ func tryToFindFileExecutePath(ps *process.Process) string { if path.Exists(exe) { return exe } - pathInNs := host.GetFileInHost(fmt.Sprintf("/proc/%d/root%s", ps.Pid, exe)) + pathInNs := host.GetHostProcInHost(fmt.Sprintf("%d/root%s", ps.Pid, exe)) if path.Exists(pathInNs) { return pathInNs } diff --git a/pkg/process/finders/kubernetes/finder.go b/pkg/process/finders/kubernetes/finder.go index 6637a7fe..03a0ba70 100644 --- a/pkg/process/finders/kubernetes/finder.go +++ b/pkg/process/finders/kubernetes/finder.go @@ -270,7 +270,7 @@ func (f *ProcessFinder) buildEntity(err error, ps *process.Process, pc *PodConta } func (f *ProcessFinder) getProcessCGroup(pid int32) ([]string, error) { - processCgroupFilePath := host.GetFileInHost(fmt.Sprintf("/proc/%d/cgroup", pid)) + processCgroupFilePath := host.GetHostProcInHost(fmt.Sprintf("%d/cgroup", pid)) cgroupFile, err := os.Open(processCgroupFilePath) if err != nil { return nil, err diff --git a/pkg/tools/btf/check.go b/pkg/tools/btf/check.go deleted file mode 100644 index 9cc3afef..00000000 --- a/pkg/tools/btf/check.go +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to Apache Software Foundation (ASF) under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Apache Software Foundation (ASF) licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package btf - -import ( - "debug/elf" - "fmt" - - "github.com/apache/skywalking-rover/pkg/tools/host" - "github.com/apache/skywalking-rover/pkg/tools/operator" - "github.com/apache/skywalking-rover/pkg/tools/path" -) - -func ExistKernelBTF() (string, error) { - if path.Exists(host.GetFileInHost("/sys/kernel/btf/vmlinux")) { - return "/sys/kernel/btf/vmlinux", nil - } - - uname, err := operator.GetOSUname() - if err != nil { - return "", err - } - - // use same list of locations as libbpf - // https://github.com/libbpf/libbpf/blob/9a3a42608dbe3731256a5682a125ac1e23bced8f/src/btf.c#L3114-L3122 - locations := []string{ - "/boot/vmlinux-%s", - "/lib/modules/%s/vmlinux-%[1]s", - "/lib/modules/%s/build/vmlinux", - "/usr/lib/modules/%s/kernel/vmlinux", - "/usr/lib/debug/boot/vmlinux-%s", - "/usr/lib/debug/boot/vmlinux-%s.debug", - "/usr/lib/debug/lib/modules/%s/vmlinux", - } - - for _, loc := range locations { - _, err := elf.Open(host.GetFileInHost(fmt.Sprintf(loc, uname.Release))) - if err == nil { - return loc, nil - } - } - return "", fmt.Errorf("could not found") -} diff --git a/pkg/tools/host/file.go b/pkg/tools/host/file.go index 63f6de37..ac02ec1a 100644 --- a/pkg/tools/host/file.go +++ b/pkg/tools/host/file.go @@ -19,23 +19,37 @@ package host import ( "os" - "strings" + "path" ) -var hostMappingPath string +var ( + hostProcMappingPath string + hostEtcMappingPath string +) func init() { - hostMappingPath = os.Getenv("ROVER_HOST_MAPPING") + hostProcMappingPath = os.Getenv("ROVER_HOST_PROC_MAPPING") // adapt with gopsutil framework to read the right process directory of host - if hostMappingPath != "" { - os.Setenv("HOST_PROC", hostMappingPath+"/proc") + if hostProcMappingPath != "" { + os.Setenv("HOST_PROC", hostProcMappingPath) + } + hostEtcMappingPath = os.Getenv("ROVER_HOST_ETC_MAPPING") +} + +func GetHostProcInHost(procSubPath string) string { + if hostProcMappingPath != "" { + return cleanPath(hostProcMappingPath + "/" + procSubPath) } + return cleanPath("/proc/" + procSubPath) } -// GetFileInHost means add the host root mapping prefix, it's dependent when the rover is deploy in a container -func GetFileInHost(absPath string) string { - if hostMappingPath != "" && strings.HasPrefix(absPath, hostMappingPath) { - return absPath +func GetHostEtcInHost(etcSubPath string) string { + if hostEtcMappingPath != "" { + return cleanPath(hostEtcMappingPath + "/" + etcSubPath) } - return hostMappingPath + absPath + return cleanPath("/etc/" + etcSubPath) +} + +func cleanPath(p string) string { + return path.Clean(p) } diff --git a/pkg/tools/operator/distribution.go b/pkg/tools/operator/distribution.go index 4ea93041..423c8bad 100644 --- a/pkg/tools/operator/distribution.go +++ b/pkg/tools/operator/distribution.go @@ -38,8 +38,8 @@ type DistributionInfo struct { // GetDistributionInfo of machine func GetDistributionInfo() (*DistributionInfo, error) { var result = &DistributionInfo{} - tryingToFindDistributionByReleaseFile(result, "/etc/lsb-release", "DISTRIB_ID", "DISTRIB_RELEASE", "") - tryingToFindDistributionByReleaseFile(result, "/etc/os-release", "ID", "VERSION_ID", "") + tryingToFindDistributionByReleaseFile(result, "lsb-release", "DISTRIB_ID", "DISTRIB_RELEASE", "") + tryingToFindDistributionByReleaseFile(result, "os-release", "ID", "VERSION_ID", "") tryingToFindDistributionByCommand(result, "Distributor ID", "Release", "", "lsb_release", "-a") tryingToFindDistributionByCommand(result, "", "", "Architecture", "hostnamectl") @@ -56,11 +56,11 @@ func GetDistributionInfo() (*DistributionInfo, error) { return result, nil } -func tryingToFindDistributionByReleaseFile(data *DistributionInfo, filename, nameKey, versionKey, architectureKey string) { +func tryingToFindDistributionByReleaseFile(data *DistributionInfo, etcSubFilename, nameKey, versionKey, architectureKey string) { if data.AllDataSuccess() { return } - file, err := os.Open(host.GetFileInHost(filename)) + file, err := os.Open(host.GetHostEtcInHost(etcSubFilename)) if err != nil { return } diff --git a/pkg/tools/process/process.go b/pkg/tools/process/process.go index b2b79d9e..87930e21 100644 --- a/pkg/tools/process/process.go +++ b/pkg/tools/process/process.go @@ -55,10 +55,10 @@ var ( // KernelFileProfilingStat is works for read the kernel and get is support for kernel symbol analyze func KernelFileProfilingStat() (*profiling.Info, error) { - if !kernelFinder.IsSupport(profiling.KernelSymbolFilePath) { + if !kernelFinder.IsSupport(profiling.KernelProcSymbolFilePath) { return nil, fmt.Errorf("not support kernel space profiling") } - return kernelFinder.Analyze(profiling.KernelSymbolFilePath) + return kernelFinder.Analyze(profiling.KernelProcSymbolFilePath) } // ProfilingStat is validating the exe file could be profiling and get info @@ -95,7 +95,7 @@ func Modules(pid int32) ([]*profiling.Module, error) { func analyzeProfilingInfo(context *analyzeContext, pid int32) (*profiling.Info, error) { // analyze process mapping - mapFile, _ := os.Open(host2.GetFileInHost(fmt.Sprintf("/proc/%d/maps", pid))) + mapFile, _ := os.Open(host2.GetHostProcInHost(fmt.Sprintf("%d/maps", pid))) scanner := bufio.NewScanner(mapFile) modules := make(map[string]*profiling.Module) for scanner.Scan() { @@ -126,7 +126,7 @@ func analyzeProfilingInfo(context *analyzeContext, pid int32) (*profiling.Info, module.Ranges = append(module.Ranges, moduleRange) continue } - modulePath := host2.GetFileInHost(fmt.Sprintf("/proc/%d/root%s", pid, moduleName)) + modulePath := host2.GetHostProcInHost(fmt.Sprintf("%d/root%s", pid, moduleName)) if !path.Exists(modulePath) { log.Debugf("could not found the module, ignore. name: %s, path: %s", moduleName, modulePath) continue diff --git a/pkg/tools/profiling/api.go b/pkg/tools/profiling/api.go index 518da05f..d1ba207d 100644 --- a/pkg/tools/profiling/api.go +++ b/pkg/tools/profiling/api.go @@ -29,7 +29,7 @@ import ( type ModuleType int8 var ( - KernelSymbolFilePath = "/proc/kallsyms" + KernelProcSymbolFilePath = "kallsyms" // after host.GetHostProcInHost, should be "/proc/kallsyms" log = logger.GetLogger("tools", "profiling") ) diff --git a/pkg/tools/profiling/kernel.go b/pkg/tools/profiling/kernel.go index 51a75c45..df8c941e 100644 --- a/pkg/tools/profiling/kernel.go +++ b/pkg/tools/profiling/kernel.go @@ -33,12 +33,12 @@ type KernelFinder struct { } func NewKernelFinder() *KernelFinder { - stat, _ := os.Stat(host.GetFileInHost(KernelSymbolFilePath)) + stat, _ := os.Stat(host.GetHostProcInHost(KernelProcSymbolFilePath)) return &KernelFinder{kernelFileExists: stat != nil} } func (k *KernelFinder) IsSupport(filepath string) bool { - if filepath != KernelSymbolFilePath { + if filepath != KernelProcSymbolFilePath { return false } stat, _ := os.Stat(filepath) diff --git a/pkg/tools/ssl/gotls.go b/pkg/tools/ssl/gotls.go index 7520d27c..2b2b3d7c 100644 --- a/pkg/tools/ssl/gotls.go +++ b/pkg/tools/ssl/gotls.go @@ -85,7 +85,7 @@ func (r *Register) GoTLS(symbolAddrMap *ebpf.Map, write, writeRet, read, readRet if buildVersionSymbol == nil { return false, nil } - pidExeFile := host.GetFileInHost(fmt.Sprintf("/proc/%d/exe", r.pid)) + pidExeFile := host.GetHostProcInHost(fmt.Sprintf("%d/exe", r.pid)) elfFile, err := elf.NewFile(pidExeFile) if err != nil { return false, fmt.Errorf("read executable file error: %v", err) diff --git a/test/e2e/cases/access_log/rover.yaml b/test/e2e/cases/access_log/rover.yaml index e32f779a..8b6a2a5f 100644 --- a/test/e2e/cases/access_log/rover.yaml +++ b/test/e2e/cases/access_log/rover.yaml @@ -68,8 +68,8 @@ spec: - SYS_ADMIN privileged: true volumeMounts: - - name: host - mountPath: /host + - name: host-proc + mountPath: /host-proc readOnly: true - name: host-sys mountPath: /sys @@ -90,8 +90,8 @@ spec: value: "false" - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_ISTIO_APPLICATION_ACTIVE value: "false" - - name: ROVER_HOST_MAPPING - value: /host + - name: ROVER_HOST_PROC_MAPPING + value: /host-proc - name: ROVER_ACCESS_LOG_ACTIVE value: "true" - name: ROVER_ACCESS_LOG_FLUSH_PERIOD @@ -100,9 +100,9 @@ spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet volumes: - - name: host + - name: host-proc hostPath: - path: /host + path: /host/proc type: Directory - name: host-sys hostPath: diff --git a/test/e2e/cases/process/istio/rover.yaml b/test/e2e/cases/process/istio/rover.yaml index 7c8e0c20..501fe200 100644 --- a/test/e2e/cases/process/istio/rover.yaml +++ b/test/e2e/cases/process/istio/rover.yaml @@ -68,8 +68,8 @@ spec: - SYS_ADMIN privileged: true volumeMounts: - - name: host - mountPath: /host + - name: host-proc + mountPath: /host-proc readOnly: true env: - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ACTIVE @@ -89,15 +89,15 @@ spec: value: "true" - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_K8S_SERVICE_ACTIVE value: "false" - - name: ROVER_HOST_MAPPING - value: /host + - name: ROVER_HOST_PROC_MAPPING + value: /host-proc - name: ROVER_CORE_CLUSTER_NAME value: e2e hostPID: true hostNetwork: true dnsPolicy: ClusterFirstWithHostNet volumes: - - name: host + - name: host-proc hostPath: - path: /host + path: /host/proc type: Directory \ No newline at end of file diff --git a/test/e2e/cases/profiling/continuous/rover.yaml b/test/e2e/cases/profiling/continuous/rover.yaml index 2b32bf2b..fdbefb51 100644 --- a/test/e2e/cases/profiling/continuous/rover.yaml +++ b/test/e2e/cases/profiling/continuous/rover.yaml @@ -68,8 +68,8 @@ spec: - SYS_ADMIN privileged: true volumeMounts: - - name: host - mountPath: /host + - name: host-proc + mountPath: /host-proc readOnly: true env: - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ACTIVE @@ -93,13 +93,13 @@ spec: value: "test-continuous" - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_K8S_SERVICE_INSTANCE_NAME value: "test-instance" - - name: ROVER_HOST_MAPPING - value: /host + - name: ROVER_HOST_PROC_MAPPING + value: /host-proc hostPID: true hostNetwork: true dnsPolicy: ClusterFirstWithHostNet volumes: - - name: host + - name: host-proc hostPath: - path: /host + path: /host/proc type: Directory \ No newline at end of file diff --git a/test/e2e/cases/profiling/task/network/rover.yaml b/test/e2e/cases/profiling/task/network/rover.yaml index 1c543d54..9818c78c 100644 --- a/test/e2e/cases/profiling/task/network/rover.yaml +++ b/test/e2e/cases/profiling/task/network/rover.yaml @@ -68,8 +68,8 @@ spec: - SYS_ADMIN privileged: true volumeMounts: - - name: host - mountPath: /host + - name: host-proc + mountPath: /host-proc readOnly: true env: - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ACTIVE @@ -95,13 +95,13 @@ spec: value: test - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_K8S_SERVICE_PROCESS_NAME value: service - - name: ROVER_HOST_MAPPING - value: /host + - name: ROVER_HOST_PROC_MAPPING + value: /host-proc hostPID: true hostNetwork: true dnsPolicy: ClusterFirstWithHostNet volumes: - - name: host + - name: host-proc hostPath: - path: /host + path: /host/proc type: Directory \ No newline at end of file diff --git a/test/e2e/cases/profiling/task/offcpu/rover.yaml b/test/e2e/cases/profiling/task/offcpu/rover.yaml index 556bfe22..0c719742 100644 --- a/test/e2e/cases/profiling/task/offcpu/rover.yaml +++ b/test/e2e/cases/profiling/task/offcpu/rover.yaml @@ -68,8 +68,8 @@ spec: - SYS_ADMIN privileged: true volumeMounts: - - name: host - mountPath: /host + - name: host-proc + mountPath: /host-proc readOnly: true env: - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ACTIVE @@ -93,13 +93,13 @@ spec: value: "file" - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_K8S_SERVICE_INSTANCE_NAME value: "test-instance" - - name: ROVER_HOST_MAPPING - value: /host + - name: ROVER_HOST_PROC_MAPPING + value: /host-proc hostPID: true hostNetwork: true dnsPolicy: ClusterFirstWithHostNet volumes: - - name: host + - name: host-proc hostPath: - path: /host + path: /host/proc type: Directory \ No newline at end of file diff --git a/test/e2e/cases/profiling/task/oncpu/rover.yaml b/test/e2e/cases/profiling/task/oncpu/rover.yaml index 5e88bdf7..d14dd7ba 100644 --- a/test/e2e/cases/profiling/task/oncpu/rover.yaml +++ b/test/e2e/cases/profiling/task/oncpu/rover.yaml @@ -68,8 +68,8 @@ spec: - SYS_ADMIN privileged: true volumeMounts: - - name: host - mountPath: /host + - name: host-proc + mountPath: /host-proc readOnly: true env: - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ACTIVE @@ -93,13 +93,13 @@ spec: value: "sqrt" - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_K8S_SERVICE_INSTANCE_NAME value: "test-instance" - - name: ROVER_HOST_MAPPING - value: /host + - name: ROVER_HOST_PROC_MAPPING + value: /host-proc hostPID: true hostNetwork: true dnsPolicy: ClusterFirstWithHostNet volumes: - - name: host + - name: host-proc hostPath: - path: /host + path: /host/proc type: Directory \ No newline at end of file