Gosum support

extractor/filesystem/language/golang/gomod/gomod.go

@@ -16,14 +16,17 @@
 package gomod
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"go/version"
 	"io"
 	"path/filepath"
 	"strings"
 
 	"github.com/google/osv-scalibr/extractor"
 	"github.com/google/osv-scalibr/extractor/filesystem"
+	"github.com/google/osv-scalibr/log"
 	"github.com/google/osv-scalibr/plugin"
 	"github.com/google/osv-scalibr/purl"
 	"golang.org/x/exp/maps"
@@ -39,10 +42,38 @@ const (
 // including the stdlib version by using the top level go version
 //
 // The output is not sorted and will not be in a consistent order
-type Extractor struct{}
+type Extractor struct {
+	deduplicateSumDependencies bool
+}
+
+// Config contains the configuration options for the extractor.
+type Config struct {
+	// DeduplicateSumDependencies controls whether dependencies in the go.sum file
+	// should be deduplicated against those in the go.mod file.
+	//
+	// When two dependencies with the same name and version exist, one in go.mod
+	// and the other in go.sum:
+	//  - If set to true, only the dependency from go.mod will be retained.
+	//  - If set to false, both dependencies (from go.mod and go.sum) will be kept.
+	DeduplicateSumDependencies bool
+}
+
+// DefaultConfig returns the default configuration for the extractor.
+func DefaultConfig() Config {
+	return Config{
+		DeduplicateSumDependencies: true,
+	}
+}
+
+// NewDefault returns a new instance of the extractor using the default configuration.
+func NewDefault() filesystem.Extractor { return New(DefaultConfig()) }
 
 // New returns a new instance of the extractor.
-func New() filesystem.Extractor { return &Extractor{} }
+func New(cfg Config) filesystem.Extractor {
+	return &Extractor{
+		deduplicateSumDependencies: cfg.DeduplicateSumDependencies,
+	}
+}
 
 // Name of the extractor.
 func (e Extractor) Name() string { return Name }
@@ -56,6 +87,9 @@ func (e Extractor) Requirements() *plugin.Capabilities {
 }
 
 // FileRequired returns true if the specified file matches go.mod files.
+//
+// go.sum is not considered since the 'go.mod' file
+// is necessary to determine the Go version before opening it.
 func (e Extractor) FileRequired(api filesystem.FileAPI) bool {
 	return filepath.Base(api.Path()) == "go.mod"
 }
@@ -75,6 +109,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
 	type mapKey struct {
 		name    string
 		version string
+		isGoSum bool
 	}
 	packages := map[mapKey]*extractor.Inventory{}
 
@@ -120,20 +155,50 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
 		}
 	}
 
+	isGoVersionSpecified := parsedLockfile.Go != nil && parsedLockfile.Go.Version != ""
+
+	// At go 1.17 and above, the go command adds an indirect requirement for each module that provides any
+	// package imported (even indirectly) by a package or test in the main module or passed as an argument to go get.
+	//
+	// for versions below extract indirect dependencies from the go.sum file
+	if isGoVersionSpecified && version.Compare("go"+parsedLockfile.Go.Version, "go1.17") < 0 {
+		sumPackages, err := extractFromSum(input)
+		if err != nil {
+			log.Warnf("Error reading go.sum file: %s", err)
+		} else {
+			for _, p := range sumPackages {
+				packages[mapKey{name: p.Name, version: p.Version, isGoSum: true}] = p
+			}
+		}
+	}
+
 	// Add the Go stdlib as an explicit dependency.
-	if parsedLockfile.Go != nil && parsedLockfile.Go.Version != "" {
+	if isGoVersionSpecified {
 		packages[mapKey{name: "stdlib"}] = &extractor.Inventory{
 			Name:      "stdlib",
 			Version:   parsedLockfile.Go.Version,
 			Locations: []string{input.Path},
 		}
 	}
 
-	// The map values might have changed after replacement so we need to run another
-	// deduplication pass.
+	// An additional deduplication pass is required.
+	// This is necessary because the values in the map may have changed after the replacement,
+	// and to ensure that sum dependencies are deduplicated when specified.
 	dedupedPs := map[mapKey]*extractor.Inventory{}
-	for _, p := range packages {
-		dedupedPs[mapKey{name: p.Name, version: p.Version}] = p
+	for key, p := range packages {
+		keepGoSumSeparated := !e.deduplicateSumDependencies
+		s := mapKey{
+			name:    p.Name,
+			version: p.Version,
+			isGoSum: key.isGoSum && keepGoSumSeparated,
+		}
+
+		// Do not override `go.mod` dependencies with by `go.sum` ones
+		if _, ok := dedupedPs[s]; ok && key.isGoSum {
+			continue
+		}
+
+		dedupedPs[s] = p
 	}
 	return maps.Values(dedupedPs), nil
 }
@@ -152,4 +217,55 @@ func (e Extractor) Ecosystem(i *extractor.Inventory) string {
 	return "Go"
 }
 
+// extractFromSum extracts dependencies from the go.sum file.
+//
+// Below 1.17 go.mod does not contain indirect dependencies
+// but they might be in go.sum, thus we look into it as well.
+//
+// Note: This function may produce false positives, as the go.sum file might be outdated.
+func extractFromSum(input *filesystem.ScanInput) ([]*extractor.Inventory, error) {
+	goSumPath := strings.TrimSuffix(input.Path, ".mod") + ".sum"
+	f, err := input.FS.Open(goSumPath)
+	if err != nil {
+		return nil, err
+	}
+
+	scanner := bufio.NewScanner(f)
+	packages := []*extractor.Inventory{}
+
+	for lineNumber := 0; scanner.Scan(); lineNumber++ {
+		line := scanner.Text()
+
+		if line == "" {
+			continue
+		}
+
+		parts := strings.Fields(line)
+		if len(parts) != 3 {
+			return nil, fmt.Errorf("Error reading go.sum file: wrongly formatted line %s:%d", goSumPath, lineNumber)
+		}
+
+		name := parts[0]
+		version := strings.TrimPrefix(parts[1], "v")
+
+		// skip a line if the version contains "/go.mod" because lines
+		// containing "/go.mod" are duplicates used to verify the hash of the go.mod file
+		if strings.Contains(version, "/go.mod") {
+			continue
+		}
+
+		packages = append(packages, &extractor.Inventory{
+			Name:      name,
+			Version:   version,
+			Locations: []string{goSumPath},
+		})
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return packages, nil
+}
+
 var _ filesystem.Extractor = Extractor{}

extractor/filesystem/language/golang/gomod/gomod_test.go

@@ -69,7 +69,13 @@ func TestExtractor_FileRequired(t *testing.T) {
 }
 
 func TestExtractor_Extract(t *testing.T) {
-	tests := []extracttest.TestTableEntry{
+	tests := []struct {
+		ExtractorConfig *gomod.Config
+		Name            string
+		InputConfig     extracttest.ScanInputMockConfig
+		WantInventory   []*extractor.Inventory
+		WantErr         error
+	}{
 		{
 			Name: "invalid",
 			InputConfig: extracttest.ScanInputMockConfig{
@@ -256,11 +262,131 @@ func TestExtractor_Extract(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "test extractor for go > 1.16",
+			InputConfig: extracttest.ScanInputMockConfig{
+				Path: "testdata/indirect-1.23.mod",
+			},
+			WantInventory: []*extractor.Inventory{
+				{
+					Name:      "github.com/sirupsen/logrus",
+					Version:   "1.9.3",
+					Locations: []string{"testdata/indirect-1.23.mod"},
+				},
+				{
+					Name:      "golang.org/x/sys",
+					Version:   "0.0.0-20220715151400-c0bba94af5f8",
+					Locations: []string{"testdata/indirect-1.23.mod"},
+				},
+				{
+					Name:      "stdlib",
+					Version:   "1.23",
+					Locations: []string{"testdata/indirect-1.23.mod"},
+				},
+			},
+		},
+		{
+			Name: "test extractor for go <=1.16",
+			InputConfig: extracttest.ScanInputMockConfig{
+				Path: "testdata/indirect-1.16.mod",
+			},
+			WantInventory: []*extractor.Inventory{
+				{
+					Name:      "github.com/davecgh/go-spew",
+					Version:   "1.1.1",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "github.com/pmezard/go-difflib",
+					Version:   "1.0.0",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "github.com/sirupsen/logrus",
+					Version:   "1.9.3",
+					Locations: []string{"testdata/indirect-1.16.mod"},
+				},
+				{
+					Name:      "github.com/stretchr/testify",
+					Version:   "1.7.0",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "golang.org/x/sys",
+					Version:   "0.0.0-20220715151400-c0bba94af5f8",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "gopkg.in/yaml.v3",
+					Version:   "3.0.0-20200313102051-9f266ea9e77c",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "stdlib",
+					Version:   "1.16",
+					Locations: []string{"testdata/indirect-1.16.mod"},
+				},
+			},
+		},
+		{
+			Name: "test extractor for go <=1.16 without deduplication",
+			ExtractorConfig: &gomod.Config{
+				DeduplicateSumDependencies: false,
+			},
+			InputConfig: extracttest.ScanInputMockConfig{
+				Path: "testdata/indirect-1.16.mod",
+			},
+			WantInventory: []*extractor.Inventory{
+				{
+					Name:      "github.com/sirupsen/logrus",
+					Version:   "1.9.3",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "github.com/davecgh/go-spew",
+					Version:   "1.1.1",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "github.com/pmezard/go-difflib",
+					Version:   "1.0.0",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "github.com/sirupsen/logrus",
+					Version:   "1.9.3",
+					Locations: []string{"testdata/indirect-1.16.mod"},
+				},
+				{
+					Name:      "github.com/stretchr/testify",
+					Version:   "1.7.0",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "golang.org/x/sys",
+					Version:   "0.0.0-20220715151400-c0bba94af5f8",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "gopkg.in/yaml.v3",
+					Version:   "3.0.0-20200313102051-9f266ea9e77c",
+					Locations: []string{"testdata/indirect-1.16.sum"},
+				},
+				{
+					Name:      "stdlib",
+					Version:   "1.16",
+					Locations: []string{"testdata/indirect-1.16.mod"},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.Name, func(t *testing.T) {
-			extr := gomod.Extractor{}
+			extr := gomod.NewDefault()
+			if tt.ExtractorConfig != nil {
+				extr = gomod.New(*tt.ExtractorConfig)
+			}
 
 			scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig)
 			defer extracttest.CloseTestScanInput(t, scanInput)

extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.mod

@@ -0,0 +1,5 @@
+module test
+
+go 1.16
+
+require github.com/sirupsen/logrus v1.9.3

extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.sum

@@ -0,0 +1,15 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.mod

@@ -0,0 +1,7 @@
+module test
+
+go 1.23
+
+require github.com/sirupsen/logrus v1.9.3
+
+require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect

extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.sum

@@ -0,0 +1 @@
+Wrongly formatted, this file should not be used!!!

extractor/filesystem/list/list.go

@@ -126,7 +126,7 @@ var (
 	// Go extractors.
 	Go = InitMap{
 		gobinary.Name: {gobinary.NewDefault},
-		gomod.Name:    {gomod.New},
+		gomod.Name:    {gomod.NewDefault},
 	}
 	// Dart extractors.
 	Dart = InitMap{pubspec.Name: {pubspec.New}}