From 60aa3d04a1ade529407d5f5c69c882708e1fb17d Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Wed, 21 Aug 2024 12:48:28 +0600 Subject: [PATCH 1/9] feat: add filePath for filePatterns into PostAnalysisInput --- pkg/fanal/analyzer/analyzer.go | 27 ++++++++++++++++++++------- pkg/fanal/artifact/image/image.go | 6 ++++-- pkg/fanal/artifact/local/fs.go | 7 +++++-- pkg/fanal/artifact/vm/vm.go | 7 +++++-- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/pkg/fanal/analyzer/analyzer.go b/pkg/fanal/analyzer/analyzer.go index abedb4b9063..9bbd6865287 100644 --- a/pkg/fanal/analyzer/analyzer.go +++ b/pkg/fanal/analyzer/analyzer.go @@ -142,8 +142,9 @@ type AnalysisInput struct { } type PostAnalysisInput struct { - FS fs.FS - Options AnalysisOptions + FS fs.FS + Options AnalysisOptions + FilePatterns []string } type AnalysisOptions struct { @@ -455,13 +456,23 @@ func (ag AnalyzerGroup) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, lim } // RequiredPostAnalyzers returns a list of analyzer types that require the given file. -func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo) []Type { +func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo, requiredByFilePatterns map[Type][]string) []Type { if info.IsDir() { return nil } var postAnalyzerTypes []Type for _, a := range ag.postAnalyzers { - if ag.filePatternMatch(a.Type(), filePath) || a.Required(filePath, info) { + if ag.filePatternMatch(a.Type(), filePath) { + postAnalyzerTypes = append(postAnalyzerTypes, a.Type()) + + // Save filePaths for files required by filePatterns to use in PostAnalyze + filePaths := []string{filePath} + if saved, ok := requiredByFilePatterns[a.Type()]; ok { + filePaths = append(filePaths, saved...) + } + requiredByFilePatterns[a.Type()] = filePaths + } + if a.Required(filePath, info) { postAnalyzerTypes = append(postAnalyzerTypes, a.Type()) } } @@ -472,7 +483,8 @@ func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo) // and passes it to the respective post-analyzer. // The obtained results are merged into the "result". // This function may be called concurrently and must be thread-safe. -func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeFS, result *AnalysisResult, opts AnalysisOptions) error { +func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeFS, result *AnalysisResult, + opts AnalysisOptions, requiredByFilePatterns map[Type][]string) error { for _, a := range ag.postAnalyzers { fsys, ok := compositeFS.Get(a.Type()) if !ok { @@ -503,8 +515,9 @@ func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeF } res, err := a.PostAnalyze(ctx, PostAnalysisInput{ - FS: filteredFS, - Options: opts, + FS: filteredFS, + Options: opts, + FilePatterns: requiredByFilePatterns[a.Type()], }) if err != nil { return xerrors.Errorf("post analysis error: %w", err) diff --git a/pkg/fanal/artifact/image/image.go b/pkg/fanal/artifact/image/image.go index b4350b25b86..abb896e60ac 100644 --- a/pkg/fanal/artifact/image/image.go +++ b/pkg/fanal/artifact/image/image.go @@ -264,6 +264,8 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable } defer composite.Cleanup() + // Files required by file patterns grouped by analyzer types + requiredByFilePatterns := make(map[analyzer.Type][]string) // Walk a tar layer opqDirs, whFiles, err := a.walker.Walk(rc, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { if err = a.analyzer.AnalyzeFile(ctx, &wg, limit, result, "", filePath, info, opener, disabled, opts); err != nil { @@ -271,7 +273,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable } // Skip post analysis if the file is not required - analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info) + analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info, requiredByFilePatterns) if len(analyzerTypes) == 0 { return nil } @@ -295,7 +297,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable wg.Wait() // Post-analysis - if err = a.analyzer.PostAnalyze(ctx, composite, result, opts); err != nil { + if err = a.analyzer.PostAnalyze(ctx, composite, result, opts, requiredByFilePatterns); err != nil { return types.BlobInfo{}, xerrors.Errorf("post analysis error: %w", err) } diff --git a/pkg/fanal/artifact/local/fs.go b/pkg/fanal/artifact/local/fs.go index 2f5ef7fe4ec..c4e79db9746 100644 --- a/pkg/fanal/artifact/local/fs.go +++ b/pkg/fanal/artifact/local/fs.go @@ -83,6 +83,9 @@ func (a Artifact) Inspect(ctx context.Context) (artifact.Reference, error) { return artifact.Reference{}, xerrors.Errorf("failed to prepare filesystem for post analysis: %w", err) } + // Files required by file patterns grouped by analyzer types + requiredByFilePatterns := make(map[analyzer.Type][]string) + err = a.walker.Walk(a.rootPath, a.artifactOption.WalkerOption, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { dir := a.rootPath @@ -97,7 +100,7 @@ func (a Artifact) Inspect(ctx context.Context) (artifact.Reference, error) { } // Skip post analysis if the file is not required - analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info) + analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info, requiredByFilePatterns) if len(analyzerTypes) == 0 { return nil } @@ -117,7 +120,7 @@ func (a Artifact) Inspect(ctx context.Context) (artifact.Reference, error) { wg.Wait() // Post-analysis - if err = a.analyzer.PostAnalyze(ctx, composite, result, opts); err != nil { + if err = a.analyzer.PostAnalyze(ctx, composite, result, opts, requiredByFilePatterns); err != nil { return artifact.Reference{}, xerrors.Errorf("post analysis error: %w", err) } diff --git a/pkg/fanal/artifact/vm/vm.go b/pkg/fanal/artifact/vm/vm.go index 56f7a0f5fa8..668d6a7386b 100644 --- a/pkg/fanal/artifact/vm/vm.go +++ b/pkg/fanal/artifact/vm/vm.go @@ -108,6 +108,9 @@ func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobI } defer composite.Cleanup() + // Files required by file patterns grouped by analyzer types + requiredByFilePatterns := make(map[analyzer.Type][]string) + // TODO: Always walk from the root directory. Consider whether there is a need to be able to set optional err = a.walker.Walk(r, "/", a.artifactOption.WalkerOption, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { path := strings.TrimPrefix(filePath, "/") @@ -116,7 +119,7 @@ func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobI } // Skip post analysis if the file is not required - analyzerTypes := a.analyzer.RequiredPostAnalyzers(path, info) + analyzerTypes := a.analyzer.RequiredPostAnalyzers(path, info, requiredByFilePatterns) if len(analyzerTypes) == 0 { return nil } @@ -142,7 +145,7 @@ func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobI } // Post-analysis - if err = a.analyzer.PostAnalyze(ctx, composite, result, opts); err != nil { + if err = a.analyzer.PostAnalyze(ctx, composite, result, opts, requiredByFilePatterns); err != nil { return types.BlobInfo{}, xerrors.Errorf("post analysis error: %w", err) } From 9a5518f8c16ae3742ba9c2cb2fa3ad455deedd1e Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Wed, 21 Aug 2024 13:14:56 +0600 Subject: [PATCH 2/9] feat: use `input.FilePatterns` in PostAnalyze functions --- pkg/fanal/analyzer/language/c/conan/conan.go | 2 +- pkg/fanal/analyzer/language/dart/pub/pubspec.go | 3 ++- pkg/fanal/analyzer/language/dotnet/nuget/nuget.go | 3 +-- pkg/fanal/analyzer/language/golang/mod/mod.go | 2 +- pkg/fanal/analyzer/language/java/gradle/lockfile.go | 3 ++- pkg/fanal/analyzer/language/julia/pkg/pkg.go | 2 +- pkg/fanal/analyzer/language/nodejs/npm/npm.go | 3 ++- pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go | 3 ++- pkg/fanal/analyzer/language/nodejs/yarn/yarn.go | 2 +- pkg/fanal/analyzer/language/php/composer/composer.go | 2 +- pkg/fanal/analyzer/language/python/packaging/packaging.go | 3 ++- pkg/fanal/analyzer/language/python/pip/pip.go | 2 +- pkg/fanal/analyzer/language/python/poetry/poetry.go | 3 ++- pkg/fanal/analyzer/language/rust/cargo/cargo.go | 2 +- 14 files changed, 20 insertions(+), 15 deletions(-) diff --git a/pkg/fanal/analyzer/language/c/conan/conan.go b/pkg/fanal/analyzer/language/c/conan/conan.go index a32591dae7f..ed465761703 100644 --- a/pkg/fanal/analyzer/language/c/conan/conan.go +++ b/pkg/fanal/analyzer/language/c/conan/conan.go @@ -45,7 +45,7 @@ func newConanLockAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, er func (a conanLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { required := func(filePath string, d fs.DirEntry) bool { - // we need all file got from `a.Required` function (conan.lock files) and from file-patterns. + // Parse all required files: `conan.lock` (from a.Required func) + input.FilePatterns return true } diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index 30fc2dadb18..e1f192be844 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -55,7 +55,8 @@ func (a pubSpecLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostA } required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.PubSpecLock + // Parse all required files: `pubspec.lock` (from a.Required func) + input.FilePatterns + return true } err = fsutils.WalkDir(input.FS, ".", required, func(path string, _ fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go b/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go index b7400048eda..109b31b1db8 100644 --- a/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go +++ b/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go @@ -59,9 +59,8 @@ func (a *nugetLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.Pos a.logger.Debug("The nuget packages directory couldn't be found. License search disabled") } - // We saved only config and lock files in the FS, - // so we need to parse all saved files required := func(path string, d fs.DirEntry) bool { + // Parse all required files: `packages.lock.json`, `packages.config` (from a.Required func) + input.FilePatterns return true } diff --git a/pkg/fanal/analyzer/language/golang/mod/mod.go b/pkg/fanal/analyzer/language/golang/mod/mod.go index 96d40ba1c95..42773663984 100644 --- a/pkg/fanal/analyzer/language/golang/mod/mod.go +++ b/pkg/fanal/analyzer/language/golang/mod/mod.go @@ -68,7 +68,7 @@ func (a *gomodAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.GoMod + return filepath.Base(path) == types.GoMod || slices.Contains(input.FilePatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, _ io.Reader) error { diff --git a/pkg/fanal/analyzer/language/java/gradle/lockfile.go b/pkg/fanal/analyzer/language/java/gradle/lockfile.go index ce7fc2c31e5..f664c5c6823 100644 --- a/pkg/fanal/analyzer/language/java/gradle/lockfile.go +++ b/pkg/fanal/analyzer/language/java/gradle/lockfile.go @@ -49,7 +49,8 @@ func (a gradleLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn } required := func(path string, d fs.DirEntry) bool { - return a.Required(path, nil) + // Parse all required files: `*gradle.lockfile` (from a.Required func) + input.FilePatterns + return true } var apps []types.Application diff --git a/pkg/fanal/analyzer/language/julia/pkg/pkg.go b/pkg/fanal/analyzer/language/julia/pkg/pkg.go index 66d715e061b..a7fb1f601a7 100644 --- a/pkg/fanal/analyzer/language/julia/pkg/pkg.go +++ b/pkg/fanal/analyzer/language/julia/pkg/pkg.go @@ -54,7 +54,7 @@ func (a juliaAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.JuliaManifest + return filepath.Base(path) == types.JuliaManifest || slices.Contains(input.FilePatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/nodejs/npm/npm.go b/pkg/fanal/analyzer/language/nodejs/npm/npm.go index 870ba1be88e..b318e4391b7 100644 --- a/pkg/fanal/analyzer/language/nodejs/npm/npm.go +++ b/pkg/fanal/analyzer/language/nodejs/npm/npm.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "slices" "golang.org/x/xerrors" @@ -47,7 +48,7 @@ func newNpmLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, e func (a npmLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { // Parse package-lock.json required := func(path string, _ fs.DirEntry) bool { - return filepath.Base(path) == types.NpmPkgLock + return filepath.Base(path) == types.NpmPkgLock || slices.Contains(input.FilePatterns, path) } var apps []types.Application diff --git a/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go b/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go index 9c8f51a3826..4bc334fa9b8 100644 --- a/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go +++ b/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "slices" "golang.org/x/xerrors" @@ -45,7 +46,7 @@ func (a pnpmAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.PnpmLock + return filepath.Base(path) == types.PnpmLock || slices.Contains(input.FilePatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go b/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go index 984f72983ec..b880b6955b5 100644 --- a/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go +++ b/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go @@ -71,7 +71,7 @@ func (a yarnAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.YarnLock + return filepath.Base(path) == types.YarnLock || slices.Contains(input.FilePatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/php/composer/composer.go b/pkg/fanal/analyzer/language/php/composer/composer.go index d0b3f446635..88af91f215c 100644 --- a/pkg/fanal/analyzer/language/php/composer/composer.go +++ b/pkg/fanal/analyzer/language/php/composer/composer.go @@ -47,7 +47,7 @@ func (a composerAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnal var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.ComposerLock + return filepath.Base(path) == types.ComposerLock || slices.Contains(input.FilePatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/python/packaging/packaging.go b/pkg/fanal/analyzer/language/python/packaging/packaging.go index 51fd585d8a6..2d6574e5cb1 100644 --- a/pkg/fanal/analyzer/language/python/packaging/packaging.go +++ b/pkg/fanal/analyzer/language/python/packaging/packaging.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "slices" "strings" "github.com/samber/lo" @@ -65,7 +66,7 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna var apps []types.Application required := func(path string, _ fs.DirEntry) bool { - return filepath.Base(path) == "METADATA" || isEggFile(path) + return filepath.Base(path) == "METADATA" || isEggFile(path) || slices.Contains(input.FilePatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index e08a90e7a70..f204965502a 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -57,8 +57,8 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn a.logger.Warn("Unable to find python `site-packages` directory. License detection is skipped.", log.Err(err)) } - // We only saved the `requirements.txt` files required := func(_ string, _ fs.DirEntry) bool { + // Parse all required files: `conan.lock` (from a.Required func) + input.FilePatterns return true } diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry.go b/pkg/fanal/analyzer/language/python/poetry/poetry.go index 3b751f74762..245439c1dbb 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "path/filepath" + "slices" "github.com/samber/lo" "golang.org/x/xerrors" @@ -44,7 +45,7 @@ func (a poetryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.PoetryLock + return filepath.Base(path) == types.PoetryLock || slices.Contains(input.FilePatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/rust/cargo/cargo.go b/pkg/fanal/analyzer/language/rust/cargo/cargo.go index ab436bf95fe..a0e398680e8 100644 --- a/pkg/fanal/analyzer/language/rust/cargo/cargo.go +++ b/pkg/fanal/analyzer/language/rust/cargo/cargo.go @@ -57,7 +57,7 @@ func (a cargoAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.CargoLock + return filepath.Base(path) == types.CargoLock || slices.Contains(input.FilePatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error { From 28ac1caafa18119d78f4ce0161e5121ce0531dd5 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Wed, 21 Aug 2024 14:40:12 +0600 Subject: [PATCH 3/9] test: add fs test --- pkg/fanal/analyzer/analyzer_test.go | 2 +- pkg/fanal/artifact/local/fs_test.go | 184 ++++++++++++------ .../local/testdata/my-package-lock.json | 22 +++ 3 files changed, 143 insertions(+), 65 deletions(-) create mode 100644 pkg/fanal/artifact/local/testdata/my-package-lock.json diff --git a/pkg/fanal/analyzer/analyzer_test.go b/pkg/fanal/analyzer/analyzer_test.go index 313fccda7e9..ce795658e84 100644 --- a/pkg/fanal/analyzer/analyzer_test.go +++ b/pkg/fanal/analyzer/analyzer_test.go @@ -630,7 +630,7 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) { ctx := context.Background() got := new(analyzer.AnalysisResult) - err = a.PostAnalyze(ctx, composite, got, analyzer.AnalysisOptions{}) + err = a.PostAnalyze(ctx, composite, got, analyzer.AnalysisOptions{}, make(map[analyzer.Type][]string)) require.NoError(t, err) assert.Equal(t, tt.want, got) }) diff --git a/pkg/fanal/artifact/local/fs_test.go b/pkg/fanal/artifact/local/fs_test.go index ba6d2879bda..5911fa6bef9 100644 --- a/pkg/fanal/artifact/local/fs_test.go +++ b/pkg/fanal/artifact/local/fs_test.go @@ -18,6 +18,7 @@ import ( "github.com/aquasecurity/trivy/pkg/misconf" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/all" + _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/nodejs/npm" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/python/pip" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/os/alpine" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/pkg/apk" @@ -47,7 +48,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:5ba63074e071e3f0247d03dd7e544b6a75f7224ee238618482c490b36f4792dc", + BlobID: "sha256:08434f862f7e9a56a6575749dd38b1885985b959c9e234a8be1b98b741fe199c", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, OS: types.OS{ @@ -82,9 +83,9 @@ func TestArtifact_Inspect(t *testing.T) { want: artifact.Reference{ Name: "host", Type: artifact.TypeFilesystem, - ID: "sha256:5ba63074e071e3f0247d03dd7e544b6a75f7224ee238618482c490b36f4792dc", + ID: "sha256:08434f862f7e9a56a6575749dd38b1885985b959c9e234a8be1b98b741fe199c", BlobIDs: []string{ - "sha256:5ba63074e071e3f0247d03dd7e544b6a75f7224ee238618482c490b36f4792dc", + "sha256:08434f862f7e9a56a6575749dd38b1885985b959c9e234a8be1b98b741fe199c", }, }, }, @@ -98,6 +99,7 @@ func TestArtifact_Inspect(t *testing.T) { analyzer.TypeAlpine, analyzer.TypeApk, analyzer.TypePip, + analyzer.TypeNpmPkgLock, }, }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ @@ -125,7 +127,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:5ba63074e071e3f0247d03dd7e544b6a75f7224ee238618482c490b36f4792dc", + BlobID: "sha256:08434f862f7e9a56a6575749dd38b1885985b959c9e234a8be1b98b741fe199c", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, OS: types.OS{ @@ -175,7 +177,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:00e49bf14e0a8c15b2d611d8e5c231276f1e10f22b3307177e513605fd18d807", + BlobID: "sha256:8e7dab5cdac2610dddfc4f7655fb83c60959414ed79b6b4bc2db8969dee6b08b", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, Applications: []types.Application{ @@ -203,9 +205,63 @@ func TestArtifact_Inspect(t *testing.T) { want: artifact.Reference{ Name: "testdata/requirements.txt", Type: artifact.TypeFilesystem, - ID: "sha256:00e49bf14e0a8c15b2d611d8e5c231276f1e10f22b3307177e513605fd18d807", + ID: "sha256:8e7dab5cdac2610dddfc4f7655fb83c60959414ed79b6b4bc2db8969dee6b08b", BlobIDs: []string{ - "sha256:00e49bf14e0a8c15b2d611d8e5c231276f1e10f22b3307177e513605fd18d807", + "sha256:8e7dab5cdac2610dddfc4f7655fb83c60959414ed79b6b4bc2db8969dee6b08b", + }, + }, + }, + { + name: "happy path with single file got from filePatterns", + fields: fields{ + dir: "testdata/my-package-lock.json", + }, + artifactOpt: artifact.Option{ + FilePatterns: []string{ + "npm:my-.*-lock.json", + }, + }, + putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ + Args: cache.ArtifactCachePutBlobArgs{ + BlobID: "sha256:d611213b69108e725dff998cb48eabd104f0bd0723dcf560233f392eb38b2541", + BlobInfo: types.BlobInfo{ + SchemaVersion: types.BlobJSONSchemaVersion, + Applications: []types.Application{ + { + Type: "npm", + FilePath: "my-package-lock.json", + Packages: types.Packages{ + { + ID: "ms@2.1.3", + Name: "ms", + Version: "2.1.3", + Relationship: types.RelationshipDirect, + Locations: []types.Location{ + { + StartLine: 15, + EndLine: 20, + }, + }, + ExternalReferences: []types.ExternalRef{ + { + Type: types.RefOther, + URL: "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + }, + }, + }, + }, + }, + }, + }, + }, + Returns: cache.ArtifactCachePutBlobReturns{}, + }, + want: artifact.Reference{ + Name: "testdata/my-package-lock.json", + Type: artifact.TypeFilesystem, + ID: "sha256:d611213b69108e725dff998cb48eabd104f0bd0723dcf560233f392eb38b2541", + BlobIDs: []string{ + "sha256:d611213b69108e725dff998cb48eabd104f0bd0723dcf560233f392eb38b2541", }, }, }, @@ -216,7 +272,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:00e49bf14e0a8c15b2d611d8e5c231276f1e10f22b3307177e513605fd18d807", + BlobID: "sha256:8e7dab5cdac2610dddfc4f7655fb83c60959414ed79b6b4bc2db8969dee6b08b", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, Applications: []types.Application{ @@ -244,9 +300,9 @@ func TestArtifact_Inspect(t *testing.T) { want: artifact.Reference{ Name: "testdata/requirements.txt", Type: artifact.TypeFilesystem, - ID: "sha256:00e49bf14e0a8c15b2d611d8e5c231276f1e10f22b3307177e513605fd18d807", + ID: "sha256:8e7dab5cdac2610dddfc4f7655fb83c60959414ed79b6b4bc2db8969dee6b08b", BlobIDs: []string{ - "sha256:00e49bf14e0a8c15b2d611d8e5c231276f1e10f22b3307177e513605fd18d807", + "sha256:8e7dab5cdac2610dddfc4f7655fb83c60959414ed79b6b4bc2db8969dee6b08b", }, }, }, @@ -341,9 +397,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/single-failure", Type: artifact.TypeFilesystem, - ID: "sha256:4f2a334086f1d175c0ee57cd4220f20b187b456dc36bbe39a63c42b5637b2179", + ID: "sha256:a17dce21ef90889a5dea35c8cb65c8e317f91713651f1f656fc4bff2647f3f70", BlobIDs: []string{ - "sha256:4f2a334086f1d175c0ee57cd4220f20b187b456dc36bbe39a63c42b5637b2179", + "sha256:a17dce21ef90889a5dea35c8cb65c8e317f91713651f1f656fc4bff2647f3f70", }, }, }, @@ -426,9 +482,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/multiple-failures", Type: artifact.TypeFilesystem, - ID: "sha256:ff7a84de97729e169c94107a89bc9da88f5ecf94873cdbd9bf0844e1af5f5b30", + ID: "sha256:81b5c45917329d0892dc5a5ea5ec73d89aaafa8b251fa94731499f9f4f658bdf", BlobIDs: []string{ - "sha256:ff7a84de97729e169c94107a89bc9da88f5ecf94873cdbd9bf0844e1af5f5b30", + "sha256:81b5c45917329d0892dc5a5ea5ec73d89aaafa8b251fa94731499f9f4f658bdf", }, }, }, @@ -456,9 +512,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/no-results", Type: artifact.TypeFilesystem, - ID: "sha256:06406e9bb7ba09d8d24c73c0995ac3b94fc1d6ce059e5a45418d7c0ab2b6dca4", + ID: "sha256:92e6c822670c479822230f144b6806931a6a18b5499788a8bf4b460894c79ef5", BlobIDs: []string{ - "sha256:06406e9bb7ba09d8d24c73c0995ac3b94fc1d6ce059e5a45418d7c0ab2b6dca4", + "sha256:92e6c822670c479822230f144b6806931a6a18b5499788a8bf4b460894c79ef5", }, }, }, @@ -505,9 +561,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/passed", Type: artifact.TypeFilesystem, - ID: "sha256:107251e6ee7312c8c27ff04e71dd943b92021777c575971809f57b60bf41bba4", + ID: "sha256:b9af9e04f44d351d0db13cea80d2ac9c573f7987d199cb602b137f345ec33025", BlobIDs: []string{ - "sha256:107251e6ee7312c8c27ff04e71dd943b92021777c575971809f57b60bf41bba4", + "sha256:b9af9e04f44d351d0db13cea80d2ac9c573f7987d199cb602b137f345ec33025", }, }, }, @@ -571,9 +627,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/busted-relative-paths/child/main.tf", Type: artifact.TypeFilesystem, - ID: "sha256:f2f07f41dbd6816d41ce6f28b3922fcedab611b8602d95e328571afd5c53b31d", + ID: "sha256:d31def375864e60ee336e7806562f877e98f5b844d0117c70065128953d71f8d", BlobIDs: []string{ - "sha256:f2f07f41dbd6816d41ce6f28b3922fcedab611b8602d95e328571afd5c53b31d", + "sha256:d31def375864e60ee336e7806562f877e98f5b844d0117c70065128953d71f8d", }, }, }, @@ -621,9 +677,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/tfvar-outside/tf", Type: artifact.TypeFilesystem, - ID: "sha256:107251e6ee7312c8c27ff04e71dd943b92021777c575971809f57b60bf41bba4", + ID: "sha256:b9af9e04f44d351d0db13cea80d2ac9c573f7987d199cb602b137f345ec33025", BlobIDs: []string{ - "sha256:107251e6ee7312c8c27ff04e71dd943b92021777c575971809f57b60bf41bba4", + "sha256:b9af9e04f44d351d0db13cea80d2ac9c573f7987d199cb602b137f345ec33025", }, }, }, @@ -711,9 +767,9 @@ func TestTerraformMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraform/relative-paths/child", Type: artifact.TypeFilesystem, - ID: "sha256:f04c37d8e5300ce9344c795c2d4e0bb1dbef251b15538a6e0c11d6d9a86664d1", + ID: "sha256:df7adc5839d508ea2bce0bf526eb08bbb4f0bc1e4d3ebf5ce897ecfabca2edca", BlobIDs: []string{ - "sha256:f04c37d8e5300ce9344c795c2d4e0bb1dbef251b15538a6e0c11d6d9a86664d1", + "sha256:df7adc5839d508ea2bce0bf526eb08bbb4f0bc1e4d3ebf5ce897ecfabca2edca", }, }, }, @@ -830,9 +886,9 @@ func TestTerraformPlanSnapshotMisconfScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraformplan/snapshots/single-failure", Type: artifact.TypeFilesystem, - ID: "sha256:c21e15d7d0cfe7c1ef1e1933b443f781d1411b864500431302a1e45fe0950529", + ID: "sha256:4ae243c0ee816ce55140d88daade3dfb9de13b0edba931c664beb5de5a4bb3d3", BlobIDs: []string{ - "sha256:c21e15d7d0cfe7c1ef1e1933b443f781d1411b864500431302a1e45fe0950529", + "sha256:4ae243c0ee816ce55140d88daade3dfb9de13b0edba931c664beb5de5a4bb3d3", }, }, }, @@ -906,9 +962,9 @@ func TestTerraformPlanSnapshotMisconfScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraformplan/snapshots/multiple-failures", Type: artifact.TypeFilesystem, - ID: "sha256:800c9ce07be36c7f4d1a4876ecfaaa77c1d90b15f43c58eaf52ea27670afcc42", + ID: "sha256:bd0eab2a9df3aa47333bcd9a39e9476127654628b57fb0e767c3736e0da00f86", BlobIDs: []string{ - "sha256:800c9ce07be36c7f4d1a4876ecfaaa77c1d90b15f43c58eaf52ea27670afcc42", + "sha256:bd0eab2a9df3aa47333bcd9a39e9476127654628b57fb0e767c3736e0da00f86", }, }, }, @@ -946,9 +1002,9 @@ func TestTerraformPlanSnapshotMisconfScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/terraformplan/snapshots/passed", Type: artifact.TypeFilesystem, - ID: "sha256:3d90bb96d2dc0af277ab0ce28972670eb81968d00775d1e92edce54ae2d165c0", + ID: "sha256:9492a26d265bfd1ac3e1973c2dfe60619eee7bd4dd7af4d7db498c36334eaa87", BlobIDs: []string{ - "sha256:3d90bb96d2dc0af277ab0ce28972670eb81968d00775d1e92edce54ae2d165c0", + "sha256:9492a26d265bfd1ac3e1973c2dfe60619eee7bd4dd7af4d7db498c36334eaa87", }, }, }, @@ -1061,9 +1117,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/single-failure/src", Type: artifact.TypeFilesystem, - ID: "sha256:bd481a673eb07ed7b51e1ff2a6e7aca08b433d11288eb9f5e9aa2d2f482a0c16", + ID: "sha256:7ab98e9e46757e54563a1dc58dddece27612a61815ce84f940512f76aeb5a373", BlobIDs: []string{ - "sha256:bd481a673eb07ed7b51e1ff2a6e7aca08b433d11288eb9f5e9aa2d2f482a0c16", + "sha256:7ab98e9e46757e54563a1dc58dddece27612a61815ce84f940512f76aeb5a373", }, }, }, @@ -1145,9 +1201,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/multiple-failures/src", Type: artifact.TypeFilesystem, - ID: "sha256:c25676d23114b9c912067d45285cd9e662cefae5e3cc82c40f67df5fee39f92a", + ID: "sha256:9ceff8d195a22fbe61e554abebc85aabb07c9495518632a965982f76875b5fc7", BlobIDs: []string{ - "sha256:c25676d23114b9c912067d45285cd9e662cefae5e3cc82c40f67df5fee39f92a", + "sha256:9ceff8d195a22fbe61e554abebc85aabb07c9495518632a965982f76875b5fc7", }, }, }, @@ -1177,9 +1233,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/no-results/src", Type: artifact.TypeFilesystem, - ID: "sha256:522b19ad182f50b7b04217831c914df52c2d2eb1bdddb02eb9cd2b4e14c9a32b", + ID: "sha256:53de80f16641bcf3c9a51544a85d085230307b7cbab9c8dbd27765ed0f1959da", BlobIDs: []string{ - "sha256:522b19ad182f50b7b04217831c914df52c2d2eb1bdddb02eb9cd2b4e14c9a32b", + "sha256:53de80f16641bcf3c9a51544a85d085230307b7cbab9c8dbd27765ed0f1959da", }, }, }, @@ -1235,9 +1291,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/params/code/src", Type: artifact.TypeFilesystem, - ID: "sha256:40d6550292de7518fd7229f7b14803c67cbffbad3376e773ad7e6dc003846e87", + ID: "sha256:5039581be69d80b93de3d98c529d48ff62195df368b1f02bd55e0fcd2ed1b53d", BlobIDs: []string{ - "sha256:40d6550292de7518fd7229f7b14803c67cbffbad3376e773ad7e6dc003846e87", + "sha256:5039581be69d80b93de3d98c529d48ff62195df368b1f02bd55e0fcd2ed1b53d", }, }, }, @@ -1293,9 +1349,9 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/cloudformation/passed/src", Type: artifact.TypeFilesystem, - ID: "sha256:e2269b8ea44e29aedeaeea83368f879b3fb0cb97bfe46bcca4383a637280cace", + ID: "sha256:bf16efcef601f232244af2a1d7c527917d0f34667794f5289e84a81514af8d17", BlobIDs: []string{ - "sha256:e2269b8ea44e29aedeaeea83368f879b3fb0cb97bfe46bcca4383a637280cace", + "sha256:bf16efcef601f232244af2a1d7c527917d0f34667794f5289e84a81514af8d17", }, }, }, @@ -1381,9 +1437,9 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/dockerfile/single-failure/src", Type: artifact.TypeFilesystem, - ID: "sha256:3551bddb0f53fb9e0c32390e3ac33f841e3cc15a52ddbcbd9ea07f7e6d1d4437", + ID: "sha256:9659d03cf3140d1aa4a6463442f951e2b9d16a153e41bd5f3d3d4b0aa350f3ca", BlobIDs: []string{ - "sha256:3551bddb0f53fb9e0c32390e3ac33f841e3cc15a52ddbcbd9ea07f7e6d1d4437", + "sha256:9659d03cf3140d1aa4a6463442f951e2b9d16a153e41bd5f3d3d4b0aa350f3ca", }, }, }, @@ -1439,9 +1495,9 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/dockerfile/multiple-failures/src", Type: artifact.TypeFilesystem, - ID: "sha256:3551bddb0f53fb9e0c32390e3ac33f841e3cc15a52ddbcbd9ea07f7e6d1d4437", + ID: "sha256:9659d03cf3140d1aa4a6463442f951e2b9d16a153e41bd5f3d3d4b0aa350f3ca", BlobIDs: []string{ - "sha256:3551bddb0f53fb9e0c32390e3ac33f841e3cc15a52ddbcbd9ea07f7e6d1d4437", + "sha256:9659d03cf3140d1aa4a6463442f951e2b9d16a153e41bd5f3d3d4b0aa350f3ca", }, }, }, @@ -1469,9 +1525,9 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/dockerfile/no-results/src", Type: artifact.TypeFilesystem, - ID: "sha256:e57ad1b0be7370a131e1265a25ac8790bbfec2bb5867315916cf92799e5855d3", + ID: "sha256:9d03393551ed1af9bf5e87037b2ced30bf30a0c75161a1ed1d783cd6df5e98c9", BlobIDs: []string{ - "sha256:e57ad1b0be7370a131e1265a25ac8790bbfec2bb5867315916cf92799e5855d3", + "sha256:9d03393551ed1af9bf5e87037b2ced30bf30a0c75161a1ed1d783cd6df5e98c9", }, }, }, @@ -1529,9 +1585,9 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/dockerfile/passed/src", Type: artifact.TypeFilesystem, - ID: "sha256:ff4a3a7aed57bd8190277cf2cc16213eef43b7a37f26f8458525f2efd9793e8f", + ID: "sha256:63bb406c0b1662d29596fdc357ff9d8051e521620e0c827d4b29ff1efe4df90b", BlobIDs: []string{ - "sha256:ff4a3a7aed57bd8190277cf2cc16213eef43b7a37f26f8458525f2efd9793e8f", + "sha256:63bb406c0b1662d29596fdc357ff9d8051e521620e0c827d4b29ff1efe4df90b", }, }, }, @@ -1621,9 +1677,9 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/kubernetes/single-failure/src", Type: artifact.TypeFilesystem, - ID: "sha256:63ceedb6582e29ee4184b8b776ee27efe226d07a932461639c05bfbe47bf7efa", + ID: "sha256:5d6175a5c00b82ccfe1457f57ff65c193bf18c9b8ed1adbba95dab1000c9a609", BlobIDs: []string{ - "sha256:63ceedb6582e29ee4184b8b776ee27efe226d07a932461639c05bfbe47bf7efa", + "sha256:5d6175a5c00b82ccfe1457f57ff65c193bf18c9b8ed1adbba95dab1000c9a609", }, }, }, @@ -1707,9 +1763,9 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/kubernetes/multiple-failures/src", Type: artifact.TypeFilesystem, - ID: "sha256:47fcc85b182385fc6cd7ca08270efff33281ba7717c7a97c7b28a47bef24fae3", + ID: "sha256:6cac5e4862b30e92a8f01f023e38cd0d53d5cf58903674bae7d9da949da01bbc", BlobIDs: []string{ - "sha256:47fcc85b182385fc6cd7ca08270efff33281ba7717c7a97c7b28a47bef24fae3", + "sha256:6cac5e4862b30e92a8f01f023e38cd0d53d5cf58903674bae7d9da949da01bbc", }, }, }, @@ -1737,9 +1793,9 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/kubernetes/no-results/src", Type: artifact.TypeFilesystem, - ID: "sha256:4aad6cb079f406935fa383e126616cee6c82e326a92c163042d6043596f18e04", + ID: "sha256:8211cd1fe39211df971a124672cd5a3e5bab64d07a8feb30a9f122efd60486d7", BlobIDs: []string{ - "sha256:4aad6cb079f406935fa383e126616cee6c82e326a92c163042d6043596f18e04", + "sha256:8211cd1fe39211df971a124672cd5a3e5bab64d07a8feb30a9f122efd60486d7", }, }, }, @@ -1797,9 +1853,9 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/kubernetes/passed/src", Type: artifact.TypeFilesystem, - ID: "sha256:b781859c685b32a25e96e54b331957d696cedfc98162146819ac64d3f157660e", + ID: "sha256:eb8ba4a472e0447a2386131dff82f703121ed6e5eb322cad38eaa0826a8c71e4", BlobIDs: []string{ - "sha256:b781859c685b32a25e96e54b331957d696cedfc98162146819ac64d3f157660e", + "sha256:eb8ba4a472e0447a2386131dff82f703121ed6e5eb322cad38eaa0826a8c71e4", }, }, }, @@ -1886,9 +1942,9 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/azurearm/single-failure/src", Type: artifact.TypeFilesystem, - ID: "sha256:62a167d993f603f5552042e4b3c7ac3a65dbbe62bad28e72631c69c9a8f5e2b5", + ID: "sha256:c2d7f3cdf20d7bb213405b0f51377632c624efb95075e19a6c9b2272859692f7", BlobIDs: []string{ - "sha256:62a167d993f603f5552042e4b3c7ac3a65dbbe62bad28e72631c69c9a8f5e2b5", + "sha256:c2d7f3cdf20d7bb213405b0f51377632c624efb95075e19a6c9b2272859692f7", }, }, }, @@ -1968,9 +2024,9 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/azurearm/multiple-failures/src", Type: artifact.TypeFilesystem, - ID: "sha256:3cc8c966f10a75dc902589329cf202168176243ef8fdec7219452bb54d02af8e", + ID: "sha256:f96a5b1d9d9e7ccb88c6f1a2faa7b0cfc5d580112f1d3d722bb348e6be635ad3", BlobIDs: []string{ - "sha256:3cc8c966f10a75dc902589329cf202168176243ef8fdec7219452bb54d02af8e", + "sha256:f96a5b1d9d9e7ccb88c6f1a2faa7b0cfc5d580112f1d3d722bb348e6be635ad3", }, }, }, @@ -1998,9 +2054,9 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/azurearm/no-results/src", Type: artifact.TypeFilesystem, - ID: "sha256:522b19ad182f50b7b04217831c914df52c2d2eb1bdddb02eb9cd2b4e14c9a32b", + ID: "sha256:53de80f16641bcf3c9a51544a85d085230307b7cbab9c8dbd27765ed0f1959da", BlobIDs: []string{ - "sha256:522b19ad182f50b7b04217831c914df52c2d2eb1bdddb02eb9cd2b4e14c9a32b", + "sha256:53de80f16641bcf3c9a51544a85d085230307b7cbab9c8dbd27765ed0f1959da", }, }, }, @@ -2054,9 +2110,9 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { want: artifact.Reference{ Name: "testdata/misconfig/azurearm/passed/src", Type: artifact.TypeFilesystem, - ID: "sha256:d6a4722cb6865cac6f55c1789d64c57479539e9198722519918764a230586b4b", + ID: "sha256:c7c06b6d7899778b81ebcf9936a758f55df52484cb7078c86f7fe578d999280f", BlobIDs: []string{ - "sha256:d6a4722cb6865cac6f55c1789d64c57479539e9198722519918764a230586b4b", + "sha256:c7c06b6d7899778b81ebcf9936a758f55df52484cb7078c86f7fe578d999280f", }, }, }, diff --git a/pkg/fanal/artifact/local/testdata/my-package-lock.json b/pkg/fanal/artifact/local/testdata/my-package-lock.json new file mode 100644 index 00000000000..2d62c88be72 --- /dev/null +++ b/pkg/fanal/artifact/local/testdata/my-package-lock.json @@ -0,0 +1,22 @@ +{ + "name": "app", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "app", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "ms": "^2.1.3" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + } + } +} \ No newline at end of file From 5f493acb3671d7a2f53980b6e8a84c0236832613 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 3 Sep 2024 10:45:15 +0600 Subject: [PATCH 4/9] refactor: rename `FilePatterns` to `FilePathsMatchedFromPatterns` --- pkg/fanal/analyzer/analyzer.go | 12 ++++++------ pkg/fanal/analyzer/language/c/conan/conan.go | 2 +- pkg/fanal/analyzer/language/dart/pub/pubspec.go | 2 +- pkg/fanal/analyzer/language/dotnet/nuget/nuget.go | 2 +- pkg/fanal/analyzer/language/golang/mod/mod.go | 2 +- pkg/fanal/analyzer/language/java/gradle/lockfile.go | 2 +- pkg/fanal/analyzer/language/julia/pkg/pkg.go | 2 +- pkg/fanal/analyzer/language/nodejs/npm/npm.go | 2 +- pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go | 2 +- pkg/fanal/analyzer/language/nodejs/yarn/yarn.go | 2 +- pkg/fanal/analyzer/language/php/composer/composer.go | 2 +- .../analyzer/language/python/packaging/packaging.go | 2 +- pkg/fanal/analyzer/language/python/pip/pip.go | 2 +- pkg/fanal/analyzer/language/python/poetry/poetry.go | 2 +- pkg/fanal/analyzer/language/rust/cargo/cargo.go | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pkg/fanal/analyzer/analyzer.go b/pkg/fanal/analyzer/analyzer.go index 9bbd6865287..c30d835bbcd 100644 --- a/pkg/fanal/analyzer/analyzer.go +++ b/pkg/fanal/analyzer/analyzer.go @@ -142,9 +142,9 @@ type AnalysisInput struct { } type PostAnalysisInput struct { - FS fs.FS - Options AnalysisOptions - FilePatterns []string + FS fs.FS + Options AnalysisOptions + FilePathsMatchedFromPatterns []string // List of filePaths got from --file-patterns flag } type AnalysisOptions struct { @@ -515,9 +515,9 @@ func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeF } res, err := a.PostAnalyze(ctx, PostAnalysisInput{ - FS: filteredFS, - Options: opts, - FilePatterns: requiredByFilePatterns[a.Type()], + FS: filteredFS, + Options: opts, + FilePathsMatchedFromPatterns: requiredByFilePatterns[a.Type()], }) if err != nil { return xerrors.Errorf("post analysis error: %w", err) diff --git a/pkg/fanal/analyzer/language/c/conan/conan.go b/pkg/fanal/analyzer/language/c/conan/conan.go index ed465761703..a63d4a46202 100644 --- a/pkg/fanal/analyzer/language/c/conan/conan.go +++ b/pkg/fanal/analyzer/language/c/conan/conan.go @@ -45,7 +45,7 @@ func newConanLockAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, er func (a conanLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { required := func(filePath string, d fs.DirEntry) bool { - // Parse all required files: `conan.lock` (from a.Required func) + input.FilePatterns + // Parse all required files: `conan.lock` (from a.Required func) + input.FilePathsMatchedFromPatterns return true } diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index e1f192be844..c64cf4a3183 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -55,7 +55,7 @@ func (a pubSpecLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostA } required := func(path string, d fs.DirEntry) bool { - // Parse all required files: `pubspec.lock` (from a.Required func) + input.FilePatterns + // Parse all required files: `pubspec.lock` (from a.Required func) + input.FilePathsMatchedFromPatterns return true } diff --git a/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go b/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go index 109b31b1db8..75c254a45a9 100644 --- a/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go +++ b/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go @@ -60,7 +60,7 @@ func (a *nugetLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.Pos } required := func(path string, d fs.DirEntry) bool { - // Parse all required files: `packages.lock.json`, `packages.config` (from a.Required func) + input.FilePatterns + // Parse all required files: `packages.lock.json`, `packages.config` (from a.Required func) + input.FilePathsMatchedFromPatterns return true } diff --git a/pkg/fanal/analyzer/language/golang/mod/mod.go b/pkg/fanal/analyzer/language/golang/mod/mod.go index 42773663984..31653374dfa 100644 --- a/pkg/fanal/analyzer/language/golang/mod/mod.go +++ b/pkg/fanal/analyzer/language/golang/mod/mod.go @@ -68,7 +68,7 @@ func (a *gomodAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.GoMod || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == types.GoMod || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, _ io.Reader) error { diff --git a/pkg/fanal/analyzer/language/java/gradle/lockfile.go b/pkg/fanal/analyzer/language/java/gradle/lockfile.go index f664c5c6823..3e9c7419b0d 100644 --- a/pkg/fanal/analyzer/language/java/gradle/lockfile.go +++ b/pkg/fanal/analyzer/language/java/gradle/lockfile.go @@ -49,7 +49,7 @@ func (a gradleLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn } required := func(path string, d fs.DirEntry) bool { - // Parse all required files: `*gradle.lockfile` (from a.Required func) + input.FilePatterns + // Parse all required files: `*gradle.lockfile` (from a.Required func) + input.FilePathsMatchedFromPatterns return true } diff --git a/pkg/fanal/analyzer/language/julia/pkg/pkg.go b/pkg/fanal/analyzer/language/julia/pkg/pkg.go index a7fb1f601a7..7ae13e85d44 100644 --- a/pkg/fanal/analyzer/language/julia/pkg/pkg.go +++ b/pkg/fanal/analyzer/language/julia/pkg/pkg.go @@ -54,7 +54,7 @@ func (a juliaAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.JuliaManifest || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == types.JuliaManifest || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/nodejs/npm/npm.go b/pkg/fanal/analyzer/language/nodejs/npm/npm.go index b318e4391b7..f21427668d2 100644 --- a/pkg/fanal/analyzer/language/nodejs/npm/npm.go +++ b/pkg/fanal/analyzer/language/nodejs/npm/npm.go @@ -48,7 +48,7 @@ func newNpmLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, e func (a npmLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { // Parse package-lock.json required := func(path string, _ fs.DirEntry) bool { - return filepath.Base(path) == types.NpmPkgLock || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == types.NpmPkgLock || slices.Contains(input.FilePathsMatchedFromPatterns, path) } var apps []types.Application diff --git a/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go b/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go index 4bc334fa9b8..5fe3cff19e3 100644 --- a/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go +++ b/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go @@ -46,7 +46,7 @@ func (a pnpmAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.PnpmLock || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == types.PnpmLock || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go b/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go index b880b6955b5..b8afd7cef96 100644 --- a/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go +++ b/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go @@ -71,7 +71,7 @@ func (a yarnAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.YarnLock || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == types.YarnLock || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/php/composer/composer.go b/pkg/fanal/analyzer/language/php/composer/composer.go index 88af91f215c..879529bec9a 100644 --- a/pkg/fanal/analyzer/language/php/composer/composer.go +++ b/pkg/fanal/analyzer/language/php/composer/composer.go @@ -47,7 +47,7 @@ func (a composerAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnal var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.ComposerLock || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == types.ComposerLock || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/python/packaging/packaging.go b/pkg/fanal/analyzer/language/python/packaging/packaging.go index 2d6574e5cb1..05453cc636b 100644 --- a/pkg/fanal/analyzer/language/python/packaging/packaging.go +++ b/pkg/fanal/analyzer/language/python/packaging/packaging.go @@ -66,7 +66,7 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna var apps []types.Application required := func(path string, _ fs.DirEntry) bool { - return filepath.Base(path) == "METADATA" || isEggFile(path) || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == "METADATA" || isEggFile(path) || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index f204965502a..48c6ca87571 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -58,7 +58,7 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn } required := func(_ string, _ fs.DirEntry) bool { - // Parse all required files: `conan.lock` (from a.Required func) + input.FilePatterns + // Parse all required files: `conan.lock` (from a.Required func) + input.FilePathsMatchedFromPatterns return true } diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry.go b/pkg/fanal/analyzer/language/python/poetry/poetry.go index 245439c1dbb..0aede0af5e9 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry.go @@ -45,7 +45,7 @@ func (a poetryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.PoetryLock || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == types.PoetryLock || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { diff --git a/pkg/fanal/analyzer/language/rust/cargo/cargo.go b/pkg/fanal/analyzer/language/rust/cargo/cargo.go index a0e398680e8..c3fce63319a 100644 --- a/pkg/fanal/analyzer/language/rust/cargo/cargo.go +++ b/pkg/fanal/analyzer/language/rust/cargo/cargo.go @@ -57,7 +57,7 @@ func (a cargoAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi var apps []types.Application required := func(path string, d fs.DirEntry) bool { - return filepath.Base(path) == types.CargoLock || slices.Contains(input.FilePatterns, path) + return filepath.Base(path) == types.CargoLock || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error { From afb738737c72774849c3d701da92244d3b0b28ae Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 3 Sep 2024 12:18:55 +0600 Subject: [PATCH 5/9] refactor: move `.egg` files to separate analyzer --- pkg/fanal/analyzer/const.go | 9 +- .../analyzer/language/python/packaging/egg.go | 135 ++++++++++++++++++ .../language/python/packaging/egg_test.go | 123 ++++++++++++++++ .../language/python/packaging/packaging.go | 57 +------- .../python/packaging/packaging_test.go | 27 ---- 5 files changed, 266 insertions(+), 85 deletions(-) create mode 100644 pkg/fanal/analyzer/language/python/packaging/egg.go create mode 100644 pkg/fanal/analyzer/language/python/packaging/egg_test.go diff --git a/pkg/fanal/analyzer/const.go b/pkg/fanal/analyzer/const.go index ea2108e8928..197c0033296 100644 --- a/pkg/fanal/analyzer/const.go +++ b/pkg/fanal/analyzer/const.go @@ -75,10 +75,11 @@ const ( TypeCondaEnv Type = "conda-environment" // Python - TypePythonPkg Type = "python-pkg" - TypePip Type = "pip" - TypePipenv Type = "pipenv" - TypePoetry Type = "poetry" + TypePythonPkg Type = "python-pkg" + TypePythonPkgEgg Type = "python-egg" + TypePip Type = "pip" + TypePipenv Type = "pipenv" + TypePoetry Type = "poetry" // Go TypeGoBinary Type = "gobinary" diff --git a/pkg/fanal/analyzer/language/python/packaging/egg.go b/pkg/fanal/analyzer/language/python/packaging/egg.go new file mode 100644 index 00000000000..a1d2c144ad1 --- /dev/null +++ b/pkg/fanal/analyzer/language/python/packaging/egg.go @@ -0,0 +1,135 @@ +package packaging + +import ( + "archive/zip" + "bytes" + "context" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + + "github.com/samber/lo" + "golang.org/x/xerrors" + + "github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging" + "github.com/aquasecurity/trivy/pkg/fanal/analyzer" + "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" + "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/utils/fsutils" + xio "github.com/aquasecurity/trivy/pkg/x/io" +) + +func init() { + analyzer.RegisterPostAnalyzer(analyzer.TypePythonPkgEgg, newEggAnalyzer) +} + +const ( + eggAnalyzerVersion = 1 + eggExt = ".egg" +) + +func newEggAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { + return &eggAnalyzer{ + logger: log.WithPrefix("python"), + pkgParser: packaging.NewParser(), + }, nil +} + +type eggAnalyzer struct { + logger *log.Logger + pkgParser language.Parser +} + +// PostAnalyze analyzes egg archive files +func (a eggAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { + var apps []types.Application + + required := func(path string, _ fs.DirEntry) bool { + return a.Required(path, nil) || slices.Contains(input.FilePathsMatchedFromPatterns, path) + } + + err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { + rsa, ok := r.(xio.ReadSeekerAt) + if !ok { + return xerrors.New("invalid reader") + } + + // .egg file is zip format and PKG-INFO needs to be extracted from the zip file. + info, err := d.Info() + if err != nil { + return xerrors.Errorf("egg file error: %w", err) + } + pkginfoInZip, err := a.analyzeEggZip(rsa, info.Size()) + if err != nil { + return xerrors.Errorf("egg analysis error: %w", err) + } + + // Egg archive may not contain required files, then we will get nil. Skip this archives + if pkginfoInZip == nil { + return nil + } + + app, err := language.ParsePackage(types.PythonPkg, path, pkginfoInZip, a.pkgParser, input.Options.FileChecksum) + if err != nil { + return xerrors.Errorf("parse error: %w", err) + } else if app == nil { + return nil + } + + apps = append(apps, *app) + return nil + }) + + if err != nil { + return nil, xerrors.Errorf("python package walk error: %w", err) + } + return &analyzer.AnalysisResult{ + Applications: apps, + }, nil +} + +func (a eggAnalyzer) analyzeEggZip(r xio.ReadSeekerAt, size int64) (xio.ReadSeekerAt, error) { + zr, err := zip.NewReader(r, size) + if err != nil { + return nil, xerrors.Errorf("zip reader error: %w", err) + } + + found, ok := lo.Find(zr.File, func(f *zip.File) bool { + return isEggFile(f.Name) + }) + if !ok { + return nil, nil + } + return a.open(found) +} + +// open reads the file content in the zip archive to make it seekable. +func (a eggAnalyzer) open(file *zip.File) (xio.ReadSeekerAt, error) { + f, err := file.Open() + if err != nil { + return nil, err + } + defer f.Close() + + b, err := io.ReadAll(f) + if err != nil { + return nil, xerrors.Errorf("file %s open error: %w", file.Name, err) + } + + return bytes.NewReader(b), nil +} + +func (a eggAnalyzer) Required(filePath string, _ os.FileInfo) bool { + return filepath.Ext(filePath) == eggExt +} + +func (a eggAnalyzer) Type() analyzer.Type { + return analyzer.TypePythonPkgEgg +} + +func (a eggAnalyzer) Version() int { + return eggAnalyzerVersion +} diff --git a/pkg/fanal/analyzer/language/python/packaging/egg_test.go b/pkg/fanal/analyzer/language/python/packaging/egg_test.go new file mode 100644 index 00000000000..b7ea7174eda --- /dev/null +++ b/pkg/fanal/analyzer/language/python/packaging/egg_test.go @@ -0,0 +1,123 @@ +package packaging + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aquasecurity/trivy/pkg/fanal/analyzer" + "github.com/aquasecurity/trivy/pkg/fanal/types" +) + +func Test_eggAnalyzer_Analyze(t *testing.T) { + tests := []struct { + name string + dir string + includeChecksum bool + want *analyzer.AnalysisResult + wantErr string + }{ + { + name: "egg zip", + dir: "testdata/egg-zip", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.PythonPkg, + FilePath: "kitchen-1.2.6-py2.7.egg", + Packages: types.Packages{ + { + Name: "kitchen", + Version: "1.2.6", + Licenses: []string{ + "GNU Library or Lesser General Public License (LGPL)", + }, + FilePath: "kitchen-1.2.6-py2.7.egg", + }, + }, + }, + }, + }, + }, + { + name: "egg zip with checksum", + dir: "testdata/egg-zip", + includeChecksum: true, + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.PythonPkg, + FilePath: "kitchen-1.2.6-py2.7.egg", + Packages: types.Packages{ + { + Name: "kitchen", + Version: "1.2.6", + Licenses: []string{ + "GNU Library or Lesser General Public License (LGPL)", + }, + FilePath: "kitchen-1.2.6-py2.7.egg", + Digest: "sha1:4e13b6e379966771e896ee43cf8e240bf6083dca", + }, + }, + }, + }, + }, + }, + { + name: "egg zip doesn't contain required files", + dir: "testdata/no-req-files", + want: &analyzer.AnalysisResult{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + a, err := newEggAnalyzer(analyzer.AnalyzerOptions{}) + require.NoError(t, err) + got, err := a.PostAnalyze(context.Background(), analyzer.PostAnalysisInput{ + FS: os.DirFS(tt.dir), + Options: analyzer.AnalysisOptions{ + FileChecksum: tt.includeChecksum, + }, + }) + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + +} + +func Test_eggAnalyzer_Required(t *testing.T) { + tests := []struct { + name string + filePath string + want bool + }{ + { + name: "egg zip", + filePath: "python2.7/site-packages/cssutils-1.0-py2.7.egg", + want: true, + }, + { + name: "egg-info PKG-INFO", + filePath: "python3.8/site-packages/wrapt-1.12.1.egg-info/PKG-INFO", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := eggAnalyzer{} + got := a.Required(tt.filePath, nil) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/fanal/analyzer/language/python/packaging/packaging.go b/pkg/fanal/analyzer/language/python/packaging/packaging.go index 05453cc636b..a030aff1a17 100644 --- a/pkg/fanal/analyzer/language/python/packaging/packaging.go +++ b/pkg/fanal/analyzer/language/python/packaging/packaging.go @@ -1,8 +1,6 @@ package packaging import ( - "archive/zip" - "bytes" "context" "errors" "io" @@ -30,7 +28,7 @@ func init() { analyzer.RegisterPostAnalyzer(analyzer.TypePythonPkg, newPackagingAnalyzer) } -const version = 1 +const version = 2 func newPackagingAnalyzer(opt analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { return &packagingAnalyzer{ @@ -44,7 +42,7 @@ var ( eggFiles = []string{ // .egg format // https://setuptools.readthedocs.io/en/latest/deprecated/python_eggs.html#eggs-and-their-formats - ".egg", // zip format + // ".egg" is zip format. We check it in `eggAnalyzer`. "EGG-INFO/PKG-INFO", // .egg-info format: .egg-info can be a file or directory @@ -75,24 +73,6 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna return xerrors.New("invalid reader") } - // .egg file is zip format and PKG-INFO needs to be extracted from the zip file. - if strings.HasSuffix(path, ".egg") { - info, err := d.Info() - if err != nil { - return xerrors.Errorf("egg file error: %w", err) - } - pkginfoInZip, err := a.analyzeEggZip(rsa, info.Size()) - if err != nil { - return xerrors.Errorf("egg analysis error: %w", err) - } - - // Egg archive may not contain required files, then we will get nil. Skip this archives - if pkginfoInZip == nil { - return nil - } - rsa = pkginfoInZip - } - app, err := a.parse(path, rsa, input.Options.FileChecksum) if err != nil { return xerrors.Errorf("parse error: %w", err) @@ -100,7 +80,7 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna return nil } - if err := a.fillAdditionalData(input.FS, app); err != nil { + if err = a.fillAdditionalData(input.FS, app); err != nil { a.logger.Warn("Unable to collect additional info", log.Err(err)) } @@ -173,37 +153,6 @@ func (a packagingAnalyzer) parse(filePath string, r xio.ReadSeekerAt, checksum b return language.ParsePackage(types.PythonPkg, filePath, r, a.pkgParser, checksum) } -func (a packagingAnalyzer) analyzeEggZip(r io.ReaderAt, size int64) (xio.ReadSeekerAt, error) { - zr, err := zip.NewReader(r, size) - if err != nil { - return nil, xerrors.Errorf("zip reader error: %w", err) - } - - found, ok := lo.Find(zr.File, func(f *zip.File) bool { - return isEggFile(f.Name) - }) - if !ok { - return nil, nil - } - return a.open(found) -} - -// open reads the file content in the zip archive to make it seekable. -func (a packagingAnalyzer) open(file *zip.File) (xio.ReadSeekerAt, error) { - f, err := file.Open() - if err != nil { - return nil, err - } - defer f.Close() - - b, err := io.ReadAll(f) - if err != nil { - return nil, xerrors.Errorf("file %s open error: %w", file.Name, err) - } - - return bytes.NewReader(b), nil -} - func (a packagingAnalyzer) Required(filePath string, _ os.FileInfo) bool { return strings.Contains(filePath, ".dist-info") || isEggFile(filePath) } diff --git a/pkg/fanal/analyzer/language/python/packaging/packaging_test.go b/pkg/fanal/analyzer/language/python/packaging/packaging_test.go index c3a89ad0cd1..eb1a62093cf 100644 --- a/pkg/fanal/analyzer/language/python/packaging/packaging_test.go +++ b/pkg/fanal/analyzer/language/python/packaging/packaging_test.go @@ -20,28 +20,6 @@ func Test_packagingAnalyzer_Analyze(t *testing.T) { want *analyzer.AnalysisResult wantErr string }{ - { - name: "egg zip", - dir: "testdata/egg-zip", - want: &analyzer.AnalysisResult{ - Applications: []types.Application{ - { - Type: types.PythonPkg, - FilePath: "kitchen-1.2.6-py2.7.egg", - Packages: types.Packages{ - { - Name: "kitchen", - Version: "1.2.6", - Licenses: []string{ - "GNU Library or Lesser General Public License (LGPL)", - }, - FilePath: "kitchen-1.2.6-py2.7.egg", - }, - }, - }, - }, - }, - }, { name: "egg-info", dir: "testdata/happy-egg", @@ -124,11 +102,6 @@ func Test_packagingAnalyzer_Analyze(t *testing.T) { }, }, }, - { - name: "egg zip doesn't contain required files", - dir: "testdata/no-req-files", - want: &analyzer.AnalysisResult{}, - }, { name: "license file in dist.info", dir: "testdata/license-file-dist", From 6e604a5eda97d599571596e4707f0070da0ea97e Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Tue, 3 Sep 2024 13:16:45 +0600 Subject: [PATCH 6/9] feat: add license detection from file for `.egg` archives --- .../analyzer/language/python/packaging/egg.go | 67 +++++++++++++++--- .../language/python/packaging/egg_test.go | 22 ++++++ .../sample_package.egg | Bin 0 -> 1135 bytes 3 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 pkg/fanal/analyzer/language/python/packaging/testdata/egg-zip-with-license-file/sample_package.egg diff --git a/pkg/fanal/analyzer/language/python/packaging/egg.go b/pkg/fanal/analyzer/language/python/packaging/egg.go index a1d2c144ad1..387d01e71f0 100644 --- a/pkg/fanal/analyzer/language/python/packaging/egg.go +++ b/pkg/fanal/analyzer/language/python/packaging/egg.go @@ -7,8 +7,10 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "slices" + "strings" "github.com/samber/lo" "golang.org/x/xerrors" @@ -17,6 +19,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/licensing" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/utils/fsutils" xio "github.com/aquasecurity/trivy/pkg/x/io" @@ -31,16 +34,18 @@ const ( eggExt = ".egg" ) -func newEggAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { +func newEggAnalyzer(opts analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { return &eggAnalyzer{ - logger: log.WithPrefix("python"), - pkgParser: packaging.NewParser(), + logger: log.WithPrefix("python"), + pkgParser: packaging.NewParser(), + licenseClassifierConfidenceLevel: opts.LicenseScannerOption.ClassifierConfidenceLevel, }, nil } type eggAnalyzer struct { - logger *log.Logger - pkgParser language.Parser + logger *log.Logger + pkgParser language.Parser + licenseClassifierConfidenceLevel float64 } // PostAnalyze analyzes egg archive files @@ -62,7 +67,7 @@ func (a eggAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisI if err != nil { return xerrors.Errorf("egg file error: %w", err) } - pkginfoInZip, err := a.analyzeEggZip(rsa, info.Size()) + pkginfoInZip, err := a.findFileInZip(rsa, info.Size(), isEggFile) if err != nil { return xerrors.Errorf("egg analysis error: %w", err) } @@ -79,6 +84,10 @@ func (a eggAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisI return nil } + if err = a.fillLicensesFromFile(rsa, info.Size(), app); err != nil { + a.logger.Warn("Unable to fill licenses", log.FilePath(path), log.Err(err)) + } + apps = append(apps, *app) return nil }) @@ -91,14 +100,18 @@ func (a eggAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisI }, nil } -func (a eggAnalyzer) analyzeEggZip(r xio.ReadSeekerAt, size int64) (xio.ReadSeekerAt, error) { +func (a eggAnalyzer) findFileInZip(r xio.ReadSeekerAt, size int64, required func(filePath string) bool) (xio.ReadSeekerAt, error) { + if _, err := r.Seek(0, io.SeekStart); err != nil { + return nil, xerrors.Errorf("file seek error: %w", err) + } + zr, err := zip.NewReader(r, size) if err != nil { return nil, xerrors.Errorf("zip reader error: %w", err) } found, ok := lo.Find(zr.File, func(f *zip.File) bool { - return isEggFile(f.Name) + return required(f.Name) }) if !ok { return nil, nil @@ -122,6 +135,44 @@ func (a eggAnalyzer) open(file *zip.File) (xio.ReadSeekerAt, error) { return bytes.NewReader(b), nil } +func (a eggAnalyzer) fillLicensesFromFile(r xio.ReadSeekerAt, size int64, app *types.Application) error { + for i, pkg := range app.Packages { + var licenses []string + for _, license := range pkg.Licenses { + if !strings.HasPrefix(license, "file://") { + licenses = append(licenses, license) + continue + } + + required := func(filePath string) bool { + return path.Base(filePath) == path.Base(strings.TrimPrefix(license, "file://")) + } + lr, err := a.findFileInZip(r, size, required) + if err != nil { + a.logger.Debug("unable to find license file in `*.egg` file", log.Err(err)) + continue + } else if lr == nil { // zip doesn't contain license file + continue + } + + l, err := licensing.Classify("", lr, a.licenseClassifierConfidenceLevel) + if err != nil { + return xerrors.Errorf("license classify error: %w", err) + } else if l == nil { + continue + } + + // License found + foundLicenses := lo.Map(l.Findings, func(finding types.LicenseFinding, _ int) string { + return finding.Name + }) + licenses = append(licenses, foundLicenses...) + } + app.Packages[i].Licenses = licenses + } + return nil +} + func (a eggAnalyzer) Required(filePath string, _ os.FileInfo) bool { return filepath.Ext(filePath) == eggExt } diff --git a/pkg/fanal/analyzer/language/python/packaging/egg_test.go b/pkg/fanal/analyzer/language/python/packaging/egg_test.go index b7ea7174eda..ddbdf7fdc13 100644 --- a/pkg/fanal/analyzer/language/python/packaging/egg_test.go +++ b/pkg/fanal/analyzer/language/python/packaging/egg_test.go @@ -66,6 +66,28 @@ func Test_eggAnalyzer_Analyze(t *testing.T) { }, }, }, + { + name: "egg zip with license file", + dir: "testdata/egg-zip-with-license-file", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.PythonPkg, + FilePath: "sample_package.egg", + Packages: types.Packages{ + { + Name: "sample_package", + Version: "0.1", + Licenses: []string{ + "MIT", + }, + FilePath: "sample_package.egg", + }, + }, + }, + }, + }, + }, { name: "egg zip doesn't contain required files", dir: "testdata/no-req-files", diff --git a/pkg/fanal/analyzer/language/python/packaging/testdata/egg-zip-with-license-file/sample_package.egg b/pkg/fanal/analyzer/language/python/packaging/testdata/egg-zip-with-license-file/sample_package.egg new file mode 100644 index 0000000000000000000000000000000000000000..91d67dc5947b71b0d541924d68430e56d2aaa360 GIT binary patch literal 1135 zcmWIWW@Zs#VBlb2C`?t3WIzI(K(?#9yRN67o4YrYdt0HtLZ85jgn zl|j_|_MOdRGURD_U&*!KVN=0bPM7JrS}ju-yxq8|X}7fLO_$Dz58g^IeB1w|+4gg# z3478Wg~c_CV`Ao*=v;M@;90UT!DF7C&@Fu#k3T}(TSb;XlTK#0Giy&o&U)_dL9( zC`)zA-tW&?(l>HRNTy$w;hgcH>jB4Xx8LurA1}Ro*K6X1J^Q8ybMWrWJ^SQrTJX%D zX?wTlF}+SS*t~Pkq0+o%3A-QsK<@5^-1(RlBnyJzJ1@2HxTZMrNfm2vm) zRI|ek2KT-`ym*Ffx^>2$^0m1F{8rcX%#B;s1mg4EA3yY#;K_OYHR-qMq>^Ql2mKs2 z?^mw4VX#B>Ku*n!TyF_Zy{(TgO>Eo$_~N}cryQPd-phAT_{F1(j4yp2JhQre`MOK= z_oqft_9_=5oA&82&Ym~RaA`|TO|fx>abHV9VELC{>-63UoSw_Xo9#03yWx)8>FN@J z`_Bjm&8oO;*`s6h&nR~R-`ekaGIyKTY|VS6WBKX6xpeR1XOHS;zfir^e3Es_FSSOw z2mk)9dVPYQH*5b4Q-dc*D`xYTr7il-D1R^MTe-RWzW=#)iuK19PGhf4>S|uGFzA!( z5}Q`t)f?}r6$n{PcIVx(FkFuPkc4s7mzf+tkvqN0yf-oV0m*<4$47u` Date: Tue, 3 Sep 2024 13:23:07 +0600 Subject: [PATCH 7/9] refactor --- pkg/fanal/analyzer/language/python/packaging/egg.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/packaging/egg.go b/pkg/fanal/analyzer/language/python/packaging/egg.go index 387d01e71f0..7dbe18b2c6d 100644 --- a/pkg/fanal/analyzer/language/python/packaging/egg.go +++ b/pkg/fanal/analyzer/language/python/packaging/egg.go @@ -147,15 +147,15 @@ func (a eggAnalyzer) fillLicensesFromFile(r xio.ReadSeekerAt, size int64, app *t required := func(filePath string) bool { return path.Base(filePath) == path.Base(strings.TrimPrefix(license, "file://")) } - lr, err := a.findFileInZip(r, size, required) + f, err := a.findFileInZip(r, size, required) if err != nil { a.logger.Debug("unable to find license file in `*.egg` file", log.Err(err)) continue - } else if lr == nil { // zip doesn't contain license file + } else if f == nil { // zip doesn't contain license file continue } - l, err := licensing.Classify("", lr, a.licenseClassifierConfidenceLevel) + l, err := licensing.Classify("", f, a.licenseClassifierConfidenceLevel) if err != nil { return xerrors.Errorf("license classify error: %w", err) } else if l == nil { From 2b4f23b93a65d00dbc7c5bd919da3f800c04124f Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 16 Sep 2024 15:52:11 +0600 Subject: [PATCH 8/9] revert: python packaging changes --- pkg/fanal/analyzer/const.go | 9 +- .../analyzer/language/python/packaging/egg.go | 186 ------------------ .../language/python/packaging/egg_test.go | 145 -------------- .../language/python/packaging/packaging.go | 60 +++++- .../python/packaging/packaging_test.go | 27 +++ .../sample_package.egg | Bin 1135 -> 0 bytes 6 files changed, 86 insertions(+), 341 deletions(-) delete mode 100644 pkg/fanal/analyzer/language/python/packaging/egg.go delete mode 100644 pkg/fanal/analyzer/language/python/packaging/egg_test.go delete mode 100644 pkg/fanal/analyzer/language/python/packaging/testdata/egg-zip-with-license-file/sample_package.egg diff --git a/pkg/fanal/analyzer/const.go b/pkg/fanal/analyzer/const.go index 197c0033296..ea2108e8928 100644 --- a/pkg/fanal/analyzer/const.go +++ b/pkg/fanal/analyzer/const.go @@ -75,11 +75,10 @@ const ( TypeCondaEnv Type = "conda-environment" // Python - TypePythonPkg Type = "python-pkg" - TypePythonPkgEgg Type = "python-egg" - TypePip Type = "pip" - TypePipenv Type = "pipenv" - TypePoetry Type = "poetry" + TypePythonPkg Type = "python-pkg" + TypePip Type = "pip" + TypePipenv Type = "pipenv" + TypePoetry Type = "poetry" // Go TypeGoBinary Type = "gobinary" diff --git a/pkg/fanal/analyzer/language/python/packaging/egg.go b/pkg/fanal/analyzer/language/python/packaging/egg.go deleted file mode 100644 index 7dbe18b2c6d..00000000000 --- a/pkg/fanal/analyzer/language/python/packaging/egg.go +++ /dev/null @@ -1,186 +0,0 @@ -package packaging - -import ( - "archive/zip" - "bytes" - "context" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "slices" - "strings" - - "github.com/samber/lo" - "golang.org/x/xerrors" - - "github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging" - "github.com/aquasecurity/trivy/pkg/fanal/analyzer" - "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" - "github.com/aquasecurity/trivy/pkg/fanal/types" - "github.com/aquasecurity/trivy/pkg/licensing" - "github.com/aquasecurity/trivy/pkg/log" - "github.com/aquasecurity/trivy/pkg/utils/fsutils" - xio "github.com/aquasecurity/trivy/pkg/x/io" -) - -func init() { - analyzer.RegisterPostAnalyzer(analyzer.TypePythonPkgEgg, newEggAnalyzer) -} - -const ( - eggAnalyzerVersion = 1 - eggExt = ".egg" -) - -func newEggAnalyzer(opts analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { - return &eggAnalyzer{ - logger: log.WithPrefix("python"), - pkgParser: packaging.NewParser(), - licenseClassifierConfidenceLevel: opts.LicenseScannerOption.ClassifierConfidenceLevel, - }, nil -} - -type eggAnalyzer struct { - logger *log.Logger - pkgParser language.Parser - licenseClassifierConfidenceLevel float64 -} - -// PostAnalyze analyzes egg archive files -func (a eggAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { - var apps []types.Application - - required := func(path string, _ fs.DirEntry) bool { - return a.Required(path, nil) || slices.Contains(input.FilePathsMatchedFromPatterns, path) - } - - err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { - rsa, ok := r.(xio.ReadSeekerAt) - if !ok { - return xerrors.New("invalid reader") - } - - // .egg file is zip format and PKG-INFO needs to be extracted from the zip file. - info, err := d.Info() - if err != nil { - return xerrors.Errorf("egg file error: %w", err) - } - pkginfoInZip, err := a.findFileInZip(rsa, info.Size(), isEggFile) - if err != nil { - return xerrors.Errorf("egg analysis error: %w", err) - } - - // Egg archive may not contain required files, then we will get nil. Skip this archives - if pkginfoInZip == nil { - return nil - } - - app, err := language.ParsePackage(types.PythonPkg, path, pkginfoInZip, a.pkgParser, input.Options.FileChecksum) - if err != nil { - return xerrors.Errorf("parse error: %w", err) - } else if app == nil { - return nil - } - - if err = a.fillLicensesFromFile(rsa, info.Size(), app); err != nil { - a.logger.Warn("Unable to fill licenses", log.FilePath(path), log.Err(err)) - } - - apps = append(apps, *app) - return nil - }) - - if err != nil { - return nil, xerrors.Errorf("python package walk error: %w", err) - } - return &analyzer.AnalysisResult{ - Applications: apps, - }, nil -} - -func (a eggAnalyzer) findFileInZip(r xio.ReadSeekerAt, size int64, required func(filePath string) bool) (xio.ReadSeekerAt, error) { - if _, err := r.Seek(0, io.SeekStart); err != nil { - return nil, xerrors.Errorf("file seek error: %w", err) - } - - zr, err := zip.NewReader(r, size) - if err != nil { - return nil, xerrors.Errorf("zip reader error: %w", err) - } - - found, ok := lo.Find(zr.File, func(f *zip.File) bool { - return required(f.Name) - }) - if !ok { - return nil, nil - } - return a.open(found) -} - -// open reads the file content in the zip archive to make it seekable. -func (a eggAnalyzer) open(file *zip.File) (xio.ReadSeekerAt, error) { - f, err := file.Open() - if err != nil { - return nil, err - } - defer f.Close() - - b, err := io.ReadAll(f) - if err != nil { - return nil, xerrors.Errorf("file %s open error: %w", file.Name, err) - } - - return bytes.NewReader(b), nil -} - -func (a eggAnalyzer) fillLicensesFromFile(r xio.ReadSeekerAt, size int64, app *types.Application) error { - for i, pkg := range app.Packages { - var licenses []string - for _, license := range pkg.Licenses { - if !strings.HasPrefix(license, "file://") { - licenses = append(licenses, license) - continue - } - - required := func(filePath string) bool { - return path.Base(filePath) == path.Base(strings.TrimPrefix(license, "file://")) - } - f, err := a.findFileInZip(r, size, required) - if err != nil { - a.logger.Debug("unable to find license file in `*.egg` file", log.Err(err)) - continue - } else if f == nil { // zip doesn't contain license file - continue - } - - l, err := licensing.Classify("", f, a.licenseClassifierConfidenceLevel) - if err != nil { - return xerrors.Errorf("license classify error: %w", err) - } else if l == nil { - continue - } - - // License found - foundLicenses := lo.Map(l.Findings, func(finding types.LicenseFinding, _ int) string { - return finding.Name - }) - licenses = append(licenses, foundLicenses...) - } - app.Packages[i].Licenses = licenses - } - return nil -} - -func (a eggAnalyzer) Required(filePath string, _ os.FileInfo) bool { - return filepath.Ext(filePath) == eggExt -} - -func (a eggAnalyzer) Type() analyzer.Type { - return analyzer.TypePythonPkgEgg -} - -func (a eggAnalyzer) Version() int { - return eggAnalyzerVersion -} diff --git a/pkg/fanal/analyzer/language/python/packaging/egg_test.go b/pkg/fanal/analyzer/language/python/packaging/egg_test.go deleted file mode 100644 index ddbdf7fdc13..00000000000 --- a/pkg/fanal/analyzer/language/python/packaging/egg_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package packaging - -import ( - "context" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/aquasecurity/trivy/pkg/fanal/analyzer" - "github.com/aquasecurity/trivy/pkg/fanal/types" -) - -func Test_eggAnalyzer_Analyze(t *testing.T) { - tests := []struct { - name string - dir string - includeChecksum bool - want *analyzer.AnalysisResult - wantErr string - }{ - { - name: "egg zip", - dir: "testdata/egg-zip", - want: &analyzer.AnalysisResult{ - Applications: []types.Application{ - { - Type: types.PythonPkg, - FilePath: "kitchen-1.2.6-py2.7.egg", - Packages: types.Packages{ - { - Name: "kitchen", - Version: "1.2.6", - Licenses: []string{ - "GNU Library or Lesser General Public License (LGPL)", - }, - FilePath: "kitchen-1.2.6-py2.7.egg", - }, - }, - }, - }, - }, - }, - { - name: "egg zip with checksum", - dir: "testdata/egg-zip", - includeChecksum: true, - want: &analyzer.AnalysisResult{ - Applications: []types.Application{ - { - Type: types.PythonPkg, - FilePath: "kitchen-1.2.6-py2.7.egg", - Packages: types.Packages{ - { - Name: "kitchen", - Version: "1.2.6", - Licenses: []string{ - "GNU Library or Lesser General Public License (LGPL)", - }, - FilePath: "kitchen-1.2.6-py2.7.egg", - Digest: "sha1:4e13b6e379966771e896ee43cf8e240bf6083dca", - }, - }, - }, - }, - }, - }, - { - name: "egg zip with license file", - dir: "testdata/egg-zip-with-license-file", - want: &analyzer.AnalysisResult{ - Applications: []types.Application{ - { - Type: types.PythonPkg, - FilePath: "sample_package.egg", - Packages: types.Packages{ - { - Name: "sample_package", - Version: "0.1", - Licenses: []string{ - "MIT", - }, - FilePath: "sample_package.egg", - }, - }, - }, - }, - }, - }, - { - name: "egg zip doesn't contain required files", - dir: "testdata/no-req-files", - want: &analyzer.AnalysisResult{}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - a, err := newEggAnalyzer(analyzer.AnalyzerOptions{}) - require.NoError(t, err) - got, err := a.PostAnalyze(context.Background(), analyzer.PostAnalysisInput{ - FS: os.DirFS(tt.dir), - Options: analyzer.AnalysisOptions{ - FileChecksum: tt.includeChecksum, - }, - }) - - if tt.wantErr != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.wantErr) - return - } - require.NoError(t, err) - assert.Equal(t, tt.want, got) - }) - } - -} - -func Test_eggAnalyzer_Required(t *testing.T) { - tests := []struct { - name string - filePath string - want bool - }{ - { - name: "egg zip", - filePath: "python2.7/site-packages/cssutils-1.0-py2.7.egg", - want: true, - }, - { - name: "egg-info PKG-INFO", - filePath: "python3.8/site-packages/wrapt-1.12.1.egg-info/PKG-INFO", - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - a := eggAnalyzer{} - got := a.Required(tt.filePath, nil) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/pkg/fanal/analyzer/language/python/packaging/packaging.go b/pkg/fanal/analyzer/language/python/packaging/packaging.go index a030aff1a17..51fd585d8a6 100644 --- a/pkg/fanal/analyzer/language/python/packaging/packaging.go +++ b/pkg/fanal/analyzer/language/python/packaging/packaging.go @@ -1,6 +1,8 @@ package packaging import ( + "archive/zip" + "bytes" "context" "errors" "io" @@ -8,7 +10,6 @@ import ( "os" "path" "path/filepath" - "slices" "strings" "github.com/samber/lo" @@ -28,7 +29,7 @@ func init() { analyzer.RegisterPostAnalyzer(analyzer.TypePythonPkg, newPackagingAnalyzer) } -const version = 2 +const version = 1 func newPackagingAnalyzer(opt analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { return &packagingAnalyzer{ @@ -42,7 +43,7 @@ var ( eggFiles = []string{ // .egg format // https://setuptools.readthedocs.io/en/latest/deprecated/python_eggs.html#eggs-and-their-formats - // ".egg" is zip format. We check it in `eggAnalyzer`. + ".egg", // zip format "EGG-INFO/PKG-INFO", // .egg-info format: .egg-info can be a file or directory @@ -64,7 +65,7 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna var apps []types.Application required := func(path string, _ fs.DirEntry) bool { - return filepath.Base(path) == "METADATA" || isEggFile(path) || slices.Contains(input.FilePathsMatchedFromPatterns, path) + return filepath.Base(path) == "METADATA" || isEggFile(path) } err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error { @@ -73,6 +74,24 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna return xerrors.New("invalid reader") } + // .egg file is zip format and PKG-INFO needs to be extracted from the zip file. + if strings.HasSuffix(path, ".egg") { + info, err := d.Info() + if err != nil { + return xerrors.Errorf("egg file error: %w", err) + } + pkginfoInZip, err := a.analyzeEggZip(rsa, info.Size()) + if err != nil { + return xerrors.Errorf("egg analysis error: %w", err) + } + + // Egg archive may not contain required files, then we will get nil. Skip this archives + if pkginfoInZip == nil { + return nil + } + rsa = pkginfoInZip + } + app, err := a.parse(path, rsa, input.Options.FileChecksum) if err != nil { return xerrors.Errorf("parse error: %w", err) @@ -80,7 +99,7 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna return nil } - if err = a.fillAdditionalData(input.FS, app); err != nil { + if err := a.fillAdditionalData(input.FS, app); err != nil { a.logger.Warn("Unable to collect additional info", log.Err(err)) } @@ -153,6 +172,37 @@ func (a packagingAnalyzer) parse(filePath string, r xio.ReadSeekerAt, checksum b return language.ParsePackage(types.PythonPkg, filePath, r, a.pkgParser, checksum) } +func (a packagingAnalyzer) analyzeEggZip(r io.ReaderAt, size int64) (xio.ReadSeekerAt, error) { + zr, err := zip.NewReader(r, size) + if err != nil { + return nil, xerrors.Errorf("zip reader error: %w", err) + } + + found, ok := lo.Find(zr.File, func(f *zip.File) bool { + return isEggFile(f.Name) + }) + if !ok { + return nil, nil + } + return a.open(found) +} + +// open reads the file content in the zip archive to make it seekable. +func (a packagingAnalyzer) open(file *zip.File) (xio.ReadSeekerAt, error) { + f, err := file.Open() + if err != nil { + return nil, err + } + defer f.Close() + + b, err := io.ReadAll(f) + if err != nil { + return nil, xerrors.Errorf("file %s open error: %w", file.Name, err) + } + + return bytes.NewReader(b), nil +} + func (a packagingAnalyzer) Required(filePath string, _ os.FileInfo) bool { return strings.Contains(filePath, ".dist-info") || isEggFile(filePath) } diff --git a/pkg/fanal/analyzer/language/python/packaging/packaging_test.go b/pkg/fanal/analyzer/language/python/packaging/packaging_test.go index eb1a62093cf..c3a89ad0cd1 100644 --- a/pkg/fanal/analyzer/language/python/packaging/packaging_test.go +++ b/pkg/fanal/analyzer/language/python/packaging/packaging_test.go @@ -20,6 +20,28 @@ func Test_packagingAnalyzer_Analyze(t *testing.T) { want *analyzer.AnalysisResult wantErr string }{ + { + name: "egg zip", + dir: "testdata/egg-zip", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.PythonPkg, + FilePath: "kitchen-1.2.6-py2.7.egg", + Packages: types.Packages{ + { + Name: "kitchen", + Version: "1.2.6", + Licenses: []string{ + "GNU Library or Lesser General Public License (LGPL)", + }, + FilePath: "kitchen-1.2.6-py2.7.egg", + }, + }, + }, + }, + }, + }, { name: "egg-info", dir: "testdata/happy-egg", @@ -102,6 +124,11 @@ func Test_packagingAnalyzer_Analyze(t *testing.T) { }, }, }, + { + name: "egg zip doesn't contain required files", + dir: "testdata/no-req-files", + want: &analyzer.AnalysisResult{}, + }, { name: "license file in dist.info", dir: "testdata/license-file-dist", diff --git a/pkg/fanal/analyzer/language/python/packaging/testdata/egg-zip-with-license-file/sample_package.egg b/pkg/fanal/analyzer/language/python/packaging/testdata/egg-zip-with-license-file/sample_package.egg deleted file mode 100644 index 91d67dc5947b71b0d541924d68430e56d2aaa360..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1135 zcmWIWW@Zs#VBlb2C`?t3WIzI(K(?#9yRN67o4YrYdt0HtLZ85jgn zl|j_|_MOdRGURD_U&*!KVN=0bPM7JrS}ju-yxq8|X}7fLO_$Dz58g^IeB1w|+4gg# z3478Wg~c_CV`Ao*=v;M@;90UT!DF7C&@Fu#k3T}(TSb;XlTK#0Giy&o&U)_dL9( zC`)zA-tW&?(l>HRNTy$w;hgcH>jB4Xx8LurA1}Ro*K6X1J^Q8ybMWrWJ^SQrTJX%D zX?wTlF}+SS*t~Pkq0+o%3A-QsK<@5^-1(RlBnyJzJ1@2HxTZMrNfm2vm) zRI|ek2KT-`ym*Ffx^>2$^0m1F{8rcX%#B;s1mg4EA3yY#;K_OYHR-qMq>^Ql2mKs2 z?^mw4VX#B>Ku*n!TyF_Zy{(TgO>Eo$_~N}cryQPd-phAT_{F1(j4yp2JhQre`MOK= z_oqft_9_=5oA&82&Ym~RaA`|TO|fx>abHV9VELC{>-63UoSw_Xo9#03yWx)8>FN@J z`_Bjm&8oO;*`s6h&nR~R-`ekaGIyKTY|VS6WBKX6xpeR1XOHS;zfir^e3Es_FSSOw z2mk)9dVPYQH*5b4Q-dc*D`xYTr7il-D1R^MTe-RWzW=#)iuK19PGhf4>S|uGFzA!( z5}Q`t)f?}r6$n{PcIVx(FkFuPkc4s7mzf+tkvqN0yf-oV0m*<4$47u` Date: Tue, 17 Sep 2024 13:42:22 +0600 Subject: [PATCH 9/9] feat(python): use input.FilePathsMatchedFromPatterns for `packaging` PostAnalyzer --- pkg/fanal/analyzer/language/python/packaging/packaging.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/fanal/analyzer/language/python/packaging/packaging.go b/pkg/fanal/analyzer/language/python/packaging/packaging.go index 944a5abde33..83f32b8f970 100644 --- a/pkg/fanal/analyzer/language/python/packaging/packaging.go +++ b/pkg/fanal/analyzer/language/python/packaging/packaging.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "slices" "strings" "github.com/samber/lo" @@ -63,7 +64,7 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna var apps []types.Application required := func(path string, _ fs.DirEntry) bool { - return filepath.Base(path) == "METADATA" || isEggFile(path) + return filepath.Base(path) == "METADATA" || isEggFile(path) || slices.Contains(input.FilePathsMatchedFromPatterns, path) } err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {