From bdc400295d7516d5d077b4b584510c642e4de108 Mon Sep 17 00:00:00 2001 From: Kyle Quest Date: Fri, 2 Feb 2024 13:55:30 +0100 Subject: [PATCH] include-dir-bins build command flag Signed-off-by: Kyle Quest --- .../container/container_inspector.go | 2 +- pkg/app/sensor/artifact/artifact.go | 100 +++++++++++++++++- pkg/docker/dockerimage/packagefiles.go | 2 +- pkg/docker/dockerutil/dockerutil.go | 14 ++- pkg/ipc/command/command.go | 2 +- 5 files changed, 114 insertions(+), 6 deletions(-) diff --git a/pkg/app/master/inspectors/container/container_inspector.go b/pkg/app/master/inspectors/container/container_inspector.go index fd362b768f..d32c62edd8 100644 --- a/pkg/app/master/inspectors/container/container_inspector.go +++ b/pkg/app/master/inspectors/container/container_inspector.go @@ -801,7 +801,7 @@ func (i *Inspector) RunContainer() error { } if len(i.IncludeDirBinsList) > 0 { - cmd.IncludeDirBinsList = pathMapKeys(i.IncludeDirBinsList) + cmd.IncludeDirBinsList = i.IncludeDirBinsList } if len(i.IncludeExes) > 0 { diff --git a/pkg/app/sensor/artifact/artifact.go b/pkg/app/sensor/artifact/artifact.go index 7f703c58a6..ab83111b76 100644 --- a/pkg/app/sensor/artifact/artifact.go +++ b/pkg/app/sensor/artifact/artifact.go @@ -1680,6 +1680,7 @@ func (p *store) saveCertsData() { func (p *store) saveArtifacts() { var includePaths map[string]bool + var includeDirBinsList map[string]bool var newPerms map[string]*fsutil.AccessInfo syscall.Umask(0) @@ -1701,6 +1702,12 @@ func (p *store) saveArtifacts() { includePaths = map[string]bool{} } + includeDirBinsList = preparePaths(getKeys(p.cmd.IncludeDirBinsList)) + log.Debugf("saveArtifacts - includeDirBinsList(%d): %+v", len(includeDirBinsList), includeDirBinsList) + if includeDirBinsList == nil { + includeDirBinsList = map[string]bool{} + } + newPerms = getRecordsWithPerms(p.cmd.Includes) log.Debugf("saveArtifacts - newPerms(%v): %+v", len(newPerms), newPerms) @@ -2240,8 +2247,99 @@ copyIncludes: } } -copyBinIncludes: + binPathMap := map[string]struct{}{} for _, binPath := range p.cmd.IncludeBins { + binPathMap[binPath] = struct{}{} + } + +addExtraBinIncludes: + for inPath, isDir := range includeDirBinsList { + if !isDir { + log.Debugf("saveArtifacts - skipping non-directory in includeDirBinsList - %s", inPath) + continue + } + + if artifact.IsFilteredPath(inPath) { + log.Debugf("saveArtifacts - skipping filtered path in includeDirBinsList - %s", inPath) + continue + } + + for _, xpattern := range excludePatterns { + found, err := doublestar.Match(xpattern, inPath) + if err != nil { + log.Debugf("saveArtifacts - includeDirBinsList - [%s] excludePatterns Match error - %v\n", inPath, err) + //should only happen when the pattern is malformed + continue + } + if found { + log.Debugf("saveArtifacts - includeDirBinsList - [%s] - excluding (%s) ", inPath, xpattern) + continue addExtraBinIncludes + } + } + + err := filepath.Walk(inPath, + func(pth string, info os.FileInfo, err error) error { + if strings.HasPrefix(pth, "/proc/") { + log.Debugf("skipping /proc file system objects... - '%s'", pth) + return filepath.SkipDir + } + + if strings.HasPrefix(pth, "/sys/") { + log.Debugf("skipping /sys file system objects... - '%s'", pth) + return filepath.SkipDir + } + + if strings.HasPrefix(pth, "/dev/") { + log.Debugf("skipping /dev file system objects... - '%s'", pth) + return filepath.SkipDir + } + + // Optimization: Exclude folders early on to prevent slow enumerat + // Can help with mounting big folders from the host. + // TODO: Combine this logic with the similar logic in findSymlinks(). + for _, xpattern := range excludePatterns { + if match, _ := doublestar.Match(xpattern, pth); match { + if info.Mode().IsDir() { + return filepath.SkipDir + } + return nil + } + } + + if err != nil { + log.Debugf("skipping %s with error: %v", pth, err) + return nil + } + + if !info.Mode().IsRegular() { + return nil + } + + pth, err = filepath.Abs(pth) + if err != nil { + return nil + } + + if strings.HasPrefix(pth, "/proc/") || + strings.HasPrefix(pth, "/sys/") || + strings.HasPrefix(pth, "/dev/") { + return nil + } + + if binProps, _ := binfile.Detected(pth); binProps != nil && binProps.IsBin { + binPathMap[pth] = struct{}{} + } + + return nil + }) + + if err != nil { + log.Errorf("saveArtifacts - error enumerating includeDirBinsList dir (%s) - %v", inPath, err) + } + } + +copyBinIncludes: + for binPath := range binPathMap { if artifact.IsFilteredPath(binPath) { log.Debugf("saveArtifacts - skipping filtered include bin - %s", binPath) continue diff --git a/pkg/docker/dockerimage/packagefiles.go b/pkg/docker/dockerimage/packagefiles.go index 64d6e849da..0ac47439d6 100644 --- a/pkg/docker/dockerimage/packagefiles.go +++ b/pkg/docker/dockerimage/packagefiles.go @@ -391,7 +391,7 @@ func getFileMetadata(header *tar.Header) *FileMetadata { } result := &FileMetadata{ - Type: TarHeaderTypeName(header.Typeflag), + Type: ObjectTypeFromTarType(header.Typeflag), IsDir: isDir, IsDelete: isDelete, IsOpq: isOpq, diff --git a/pkg/docker/dockerutil/dockerutil.go b/pkg/docker/dockerutil/dockerutil.go index f21e87f7e9..8ad3173227 100644 --- a/pkg/docker/dockerutil/dockerutil.go +++ b/pkg/docker/dockerutil/dockerutil.go @@ -63,20 +63,30 @@ func ImageToIdentity(info *dockerapi.Image) *ImageIdentity { RepoDigests: info.RepoDigests, } + uniqueTags := map[string]struct{}{} for _, tag := range result.RepoTags { parts := strings.Split(tag, ":") if len(parts) == 2 { - result.ShortTags = append(result.ShortTags, parts[1]) + uniqueTags[parts[1]] = struct{}{} } } + for k := range uniqueTags { + result.ShortTags = append(result.ShortTags, k) + } + + uniqueDigests := map[string]struct{}{} for _, digest := range result.RepoDigests { parts := strings.Split(digest, "@") if len(parts) == 2 { - result.ShortDigests = append(result.ShortDigests, parts[1]) + uniqueDigests[parts[1]] = struct{}{} } } + for k := range uniqueDigests { + result.ShortDigests = append(result.ShortDigests, k) + } + return result } diff --git a/pkg/ipc/command/command.go b/pkg/ipc/command/command.go index 30220aad58..45b4b38edc 100644 --- a/pkg/ipc/command/command.go +++ b/pkg/ipc/command/command.go @@ -58,7 +58,7 @@ type StartMonitor struct { Preserves map[string]*fsutil.AccessInfo `json:"preserves,omitempty"` Includes map[string]*fsutil.AccessInfo `json:"includes,omitempty"` IncludeBins []string `json:"include_bins,omitempty"` - IncludeDirBinsList []string `json:"include_dir_bins_list,omitempty"` + IncludeDirBinsList map[string]*fsutil.AccessInfo `json:"include_dir_bins_list,omitempty"` IncludeExes []string `json:"include_exes,omitempty"` IncludeShell bool `json:"include_shell,omitempty"` IncludeWorkdir string `json:"include_workdir,omitempty"`