include-dir-bins build command flag

Signed-off-by: Kyle Quest <[email protected]>

pkg/app/master/inspectors/container/container_inspector.go

@@ -801,7 +801,7 @@ func (i *Inspector) RunContainer() error {
 	}
 
 	if len(i.IncludeDirBinsList) > 0 {
-		cmd.IncludeDirBinsList = pathMapKeys(i.IncludeDirBinsList)
+		cmd.IncludeDirBinsList = i.IncludeDirBinsList
 	}
 
 	if len(i.IncludeExes) > 0 {

pkg/app/sensor/artifact/artifact.go

@@ -1680,6 +1680,7 @@ func (p *store) saveCertsData() {
 
 func (p *store) saveArtifacts() {
 	var includePaths map[string]bool
+	var includeDirBinsList map[string]bool
 	var newPerms map[string]*fsutil.AccessInfo
 
 	syscall.Umask(0)
@@ -1701,6 +1702,12 @@ func (p *store) saveArtifacts() {
 		includePaths = map[string]bool{}
 	}
 
+	includeDirBinsList = preparePaths(getKeys(p.cmd.IncludeDirBinsList))
+	log.Debugf("saveArtifacts - includeDirBinsList(%d): %+v", len(includeDirBinsList), includeDirBinsList)
+	if includeDirBinsList == nil {
+		includeDirBinsList = map[string]bool{}
+	}
+
 	newPerms = getRecordsWithPerms(p.cmd.Includes)
 	log.Debugf("saveArtifacts - newPerms(%v): %+v", len(newPerms), newPerms)
 
@@ -2240,8 +2247,99 @@ copyIncludes:
 		}
 	}
 
-copyBinIncludes:
+	binPathMap := map[string]struct{}{}
 	for _, binPath := range p.cmd.IncludeBins {
+		binPathMap[binPath] = struct{}{}
+	}
+
+addExtraBinIncludes:
+	for inPath, isDir := range includeDirBinsList {
+		if !isDir {
+			log.Debugf("saveArtifacts - skipping non-directory in includeDirBinsList - %s", inPath)
+			continue
+		}
+
+		if artifact.IsFilteredPath(inPath) {
+			log.Debugf("saveArtifacts - skipping filtered path in includeDirBinsList - %s", inPath)
+			continue
+		}
+
+		for _, xpattern := range excludePatterns {
+			found, err := doublestar.Match(xpattern, inPath)
+			if err != nil {
+				log.Debugf("saveArtifacts - includeDirBinsList - [%s] excludePatterns Match error - %v\n", inPath, err)
+				//should only happen when the pattern is malformed
+				continue
+			}
+			if found {
+				log.Debugf("saveArtifacts - includeDirBinsList - [%s] - excluding (%s) ", inPath, xpattern)
+				continue addExtraBinIncludes
+			}
+		}
+
+		err := filepath.Walk(inPath,
+		func(pth string, info os.FileInfo, err error) error {
+			if strings.HasPrefix(pth, "/proc/") {
+				log.Debugf("skipping /proc file system objects... - '%s'", pth)
+				return filepath.SkipDir
+			}
+
+			if strings.HasPrefix(pth, "/sys/") {
+				log.Debugf("skipping /sys file system objects... - '%s'", pth)
+				return filepath.SkipDir
+			}
+
+			if strings.HasPrefix(pth, "/dev/") {
+				log.Debugf("skipping /dev file system objects... - '%s'", pth)
+				return filepath.SkipDir
+			}
+
+			// Optimization: Exclude folders early on to prevent slow enumerat
+			//               Can help with mounting big folders from the host.
+			// TODO: Combine this logic with the similar logic in findSymlinks().
+			for _, xpattern := range excludePatterns {
+				if match, _ := doublestar.Match(xpattern, pth); match {
+					if info.Mode().IsDir() {
+						return filepath.SkipDir
+					}
+					return nil
+				}
+			}
+
+			if err != nil {
+				log.Debugf("skipping %s with error: %v", pth, err)
+				return nil
+			}
+
+			if !info.Mode().IsRegular() {
+				return nil
+			}
+
+			pth, err = filepath.Abs(pth)
+			if err != nil {
+				return nil
+			}
+
+			if strings.HasPrefix(pth, "/proc/") ||
+				strings.HasPrefix(pth, "/sys/") ||
+				strings.HasPrefix(pth, "/dev/") {
+				return nil
+			}
+
+			if binProps, _ := binfile.Detected(pth); binProps != nil && binProps.IsBin {
+				binPathMap[pth] = struct{}{}
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			log.Errorf("saveArtifacts - error enumerating includeDirBinsList dir (%s) - %v", inPath, err)
+		}
+	}
+
+copyBinIncludes:
+	for binPath := range binPathMap {
 		if artifact.IsFilteredPath(binPath) {
 			log.Debugf("saveArtifacts - skipping filtered include bin - %s", binPath)
 			continue

pkg/docker/dockerimage/packagefiles.go

@@ -391,7 +391,7 @@ func getFileMetadata(header *tar.Header) *FileMetadata {
 	}
 
 	result := &FileMetadata{
-		Type:       TarHeaderTypeName(header.Typeflag),
+		Type:       ObjectTypeFromTarType(header.Typeflag),
 		IsDir:      isDir,
 		IsDelete:   isDelete,
 		IsOpq:      isOpq,

pkg/docker/dockerutil/dockerutil.go

@@ -63,20 +63,30 @@ func ImageToIdentity(info *dockerapi.Image) *ImageIdentity {
 		RepoDigests: info.RepoDigests,
 	}
 
+	uniqueTags := map[string]struct{}{}
 	for _, tag := range result.RepoTags {
 		parts := strings.Split(tag, ":")
 		if len(parts) == 2 {
-			result.ShortTags = append(result.ShortTags, parts[1])
+			uniqueTags[parts[1]] = struct{}{}
 		}
 	}
 
+	for k := range uniqueTags {
+		result.ShortTags = append(result.ShortTags, k)
+	}
+
+	uniqueDigests := map[string]struct{}{}
 	for _, digest := range result.RepoDigests {
 		parts := strings.Split(digest, "@")
 		if len(parts) == 2 {
-			result.ShortDigests = append(result.ShortDigests, parts[1])
+			uniqueDigests[parts[1]] = struct{}{}
 		}
 	}
 
+	for k := range uniqueDigests {
+		result.ShortDigests = append(result.ShortDigests, k)
+	}
+
 	return result
 }
 

pkg/ipc/command/command.go

@@ -58,7 +58,7 @@ type StartMonitor struct {
 	Preserves                    map[string]*fsutil.AccessInfo `json:"preserves,omitempty"`
 	Includes                     map[string]*fsutil.AccessInfo `json:"includes,omitempty"`
 	IncludeBins                  []string                      `json:"include_bins,omitempty"`
-	IncludeDirBinsList           []string                      `json:"include_dir_bins_list,omitempty"`
+	IncludeDirBinsList           map[string]*fsutil.AccessInfo `json:"include_dir_bins_list,omitempty"`
 	IncludeExes                  []string                      `json:"include_exes,omitempty"`
 	IncludeShell                 bool                          `json:"include_shell,omitempty"`
 	IncludeWorkdir               string                        `json:"include_workdir,omitempty"`