Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: use --file-patterns flag for all post analyzers #7365

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
27 changes: 20 additions & 7 deletions pkg/fanal/analyzer/analyzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,9 @@ type AnalysisInput struct {
}

type PostAnalysisInput struct {
FS fs.FS
Options AnalysisOptions
FS fs.FS
Options AnalysisOptions
FilePathsMatchedFromPatterns []string // List of filePaths got from --file-patterns flag
}

type AnalysisOptions struct {
Expand Down Expand Up @@ -455,13 +456,23 @@ func (ag AnalyzerGroup) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, lim
}

// RequiredPostAnalyzers returns a list of analyzer types that require the given file.
func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo) []Type {
func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo, requiredByFilePatterns map[Type][]string) []Type {
if info.IsDir() {
return nil
}
var postAnalyzerTypes []Type
for _, a := range ag.postAnalyzers {
if ag.filePatternMatch(a.Type(), filePath) || a.Required(filePath, info) {
if ag.filePatternMatch(a.Type(), filePath) {
postAnalyzerTypes = append(postAnalyzerTypes, a.Type())

// Save filePaths for files required by filePatterns to use in PostAnalyze
filePaths := []string{filePath}
if saved, ok := requiredByFilePatterns[a.Type()]; ok {
filePaths = append(filePaths, saved...)
}
requiredByFilePatterns[a.Type()] = filePaths
}
if a.Required(filePath, info) {
postAnalyzerTypes = append(postAnalyzerTypes, a.Type())
}
}
Expand All @@ -472,7 +483,8 @@ func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo)
// and passes it to the respective post-analyzer.
// The obtained results are merged into the "result".
// This function may be called concurrently and must be thread-safe.
func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeFS, result *AnalysisResult, opts AnalysisOptions) error {
func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeFS, result *AnalysisResult,
opts AnalysisOptions, requiredByFilePatterns map[Type][]string) error {
for _, a := range ag.postAnalyzers {
fsys, ok := compositeFS.Get(a.Type())
if !ok {
Expand Down Expand Up @@ -503,8 +515,9 @@ func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeF
}

res, err := a.PostAnalyze(ctx, PostAnalysisInput{
FS: filteredFS,
Options: opts,
FS: filteredFS,
Options: opts,
FilePathsMatchedFromPatterns: requiredByFilePatterns[a.Type()],
})
if err != nil {
return xerrors.Errorf("post analysis error: %w", err)
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/analyzer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -630,7 +630,7 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) {

ctx := context.Background()
got := new(analyzer.AnalysisResult)
err = a.PostAnalyze(ctx, composite, got, analyzer.AnalysisOptions{})
err = a.PostAnalyze(ctx, composite, got, analyzer.AnalysisOptions{}, make(map[analyzer.Type][]string))
require.NoError(t, err)
assert.Equal(t, tt.want, got)
})
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/c/conan/conan.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func newConanLockAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, er

func (a conanLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) {
required := func(filePath string, d fs.DirEntry) bool {
// we need all file got from `a.Required` function (conan.lock files) and from file-patterns.
// Parse all required files: `conan.lock` (from a.Required func) + input.FilePathsMatchedFromPatterns
return true
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/analyzer/language/dart/pub/pubspec.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ func (a pubSpecLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostA
}

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.PubSpecLock
// Parse all required files: `pubspec.lock` (from a.Required func) + input.FilePathsMatchedFromPatterns
return true
}

err = fsutils.WalkDir(input.FS, ".", required, func(path string, _ fs.DirEntry, r io.Reader) error {
Expand Down
3 changes: 1 addition & 2 deletions pkg/fanal/analyzer/language/dotnet/nuget/nuget.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,8 @@ func (a *nugetLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.Pos
a.logger.Debug("The nuget packages directory couldn't be found. License search disabled")
}

// We saved only config and lock files in the FS,
// so we need to parse all saved files
required := func(path string, d fs.DirEntry) bool {
// Parse all required files: `packages.lock.json`, `packages.config` (from a.Required func) + input.FilePathsMatchedFromPatterns
return true
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/golang/mod/mod.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func (a *gomodAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys
var apps []types.Application

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.GoMod
return filepath.Base(path) == types.GoMod || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, _ io.Reader) error {
Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/analyzer/language/java/gradle/lockfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ func (a gradleLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn
}

required := func(path string, d fs.DirEntry) bool {
return a.Required(path, nil)
// Parse all required files: `*gradle.lockfile` (from a.Required func) + input.FilePathsMatchedFromPatterns
return true
}

var apps []types.Application
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/julia/pkg/pkg.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func (a juliaAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi
var apps []types.Application

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.JuliaManifest
return filepath.Base(path) == types.JuliaManifest || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error {
Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/analyzer/language/nodejs/npm/npm.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"os"
"path"
"path/filepath"
"slices"

"golang.org/x/xerrors"

Expand Down Expand Up @@ -47,7 +48,7 @@ func newNpmLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, e
func (a npmLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) {
// Parse package-lock.json
required := func(path string, _ fs.DirEntry) bool {
return filepath.Base(path) == types.NpmPkgLock
return filepath.Base(path) == types.NpmPkgLock || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

var apps []types.Application
Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"os"
"path"
"path/filepath"
"slices"

"golang.org/x/xerrors"

Expand Down Expand Up @@ -45,7 +46,7 @@ func (a pnpmAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis
var apps []types.Application

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.PnpmLock
return filepath.Base(path) == types.PnpmLock || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/nodejs/yarn/yarn.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (a yarnAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis
var apps []types.Application

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.YarnLock
return filepath.Base(path) == types.YarnLock || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/php/composer/composer.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func (a composerAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnal
var apps []types.Application

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.ComposerLock
return filepath.Base(path) == types.ComposerLock || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error {
Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/analyzer/language/python/packaging/packaging.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"os"
"path"
"path/filepath"
"slices"
"strings"

"github.com/samber/lo"
Expand Down Expand Up @@ -63,7 +64,7 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna
var apps []types.Application

required := func(path string, _ fs.DirEntry) bool {
return filepath.Base(path) == "METADATA" || isEggFile(path)
return filepath.Base(path) == "METADATA" || isEggFile(path) || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/python/pip/pip.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn
a.logger.Warn("Unable to find python `site-packages` directory. License detection is skipped.", log.Err(err))
}

// We only saved the `requirements.txt` files
required := func(_ string, _ fs.DirEntry) bool {
// Parse all required files: `conan.lock` (from a.Required func) + input.FilePathsMatchedFromPatterns
return true
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/analyzer/language/python/poetry/poetry.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io/fs"
"os"
"path/filepath"
"slices"

"github.com/samber/lo"
"golang.org/x/xerrors"
Expand Down Expand Up @@ -44,7 +45,7 @@ func (a poetryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys
var apps []types.Application

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.PoetryLock
return filepath.Base(path) == types.PoetryLock || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error {
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/rust/cargo/cargo.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func (a cargoAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi
var apps []types.Application

required := func(path string, d fs.DirEntry) bool {
return filepath.Base(path) == types.CargoLock
return filepath.Base(path) == types.CargoLock || slices.Contains(input.FilePathsMatchedFromPatterns, path)
}

err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {
Expand Down
6 changes: 4 additions & 2 deletions pkg/fanal/artifact/image/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -264,14 +264,16 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable
}
defer composite.Cleanup()

// Files required by file patterns grouped by analyzer types
requiredByFilePatterns := make(map[analyzer.Type][]string)
// Walk a tar layer
opqDirs, whFiles, err := a.walker.Walk(rc, func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
if err = a.analyzer.AnalyzeFile(ctx, &wg, limit, result, "", filePath, info, opener, disabled, opts); err != nil {
return xerrors.Errorf("failed to analyze %s: %w", filePath, err)
}

// Skip post analysis if the file is not required
analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info)
analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info, requiredByFilePatterns)
if len(analyzerTypes) == 0 {
return nil
}
Expand All @@ -295,7 +297,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable
wg.Wait()

// Post-analysis
if err = a.analyzer.PostAnalyze(ctx, composite, result, opts); err != nil {
if err = a.analyzer.PostAnalyze(ctx, composite, result, opts, requiredByFilePatterns); err != nil {
return types.BlobInfo{}, xerrors.Errorf("post analysis error: %w", err)
}

Expand Down
7 changes: 5 additions & 2 deletions pkg/fanal/artifact/local/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ func (a Artifact) Inspect(ctx context.Context) (artifact.Reference, error) {
return artifact.Reference{}, xerrors.Errorf("failed to prepare filesystem for post analysis: %w", err)
}

// Files required by file patterns grouped by analyzer types
requiredByFilePatterns := make(map[analyzer.Type][]string)

err = a.walker.Walk(a.rootPath, a.artifactOption.WalkerOption, func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
dir := a.rootPath

Expand All @@ -97,7 +100,7 @@ func (a Artifact) Inspect(ctx context.Context) (artifact.Reference, error) {
}

// Skip post analysis if the file is not required
analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info)
analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info, requiredByFilePatterns)
if len(analyzerTypes) == 0 {
return nil
}
Expand All @@ -117,7 +120,7 @@ func (a Artifact) Inspect(ctx context.Context) (artifact.Reference, error) {
wg.Wait()

// Post-analysis
if err = a.analyzer.PostAnalyze(ctx, composite, result, opts); err != nil {
if err = a.analyzer.PostAnalyze(ctx, composite, result, opts, requiredByFilePatterns); err != nil {
return artifact.Reference{}, xerrors.Errorf("post analysis error: %w", err)
}

Expand Down
Loading