From a8ff9b6cfd55684aa36c53a800abd7b10a78b2ae Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Thu, 4 Jul 2024 15:14:34 +0530 Subject: [PATCH 01/11] add new args fs in processFile func Signed-off-by: Vivek Kumar Sahu --- pkg/engine/dtrack.go | 5 +- pkg/engine/score.go | 194 +++++++++++++++++++++++++++++++++---------- pkg/engine/share.go | 3 +- 3 files changed, 154 insertions(+), 48 deletions(-) diff --git a/pkg/engine/dtrack.go b/pkg/engine/dtrack.go index 90f04aa..dc0ef2a 100644 --- a/pkg/engine/dtrack.go +++ b/pkg/engine/dtrack.go @@ -49,7 +49,6 @@ func DtrackScore(ctx context.Context, dtP *DtParams) error { dTrackClient, err := dtrack.NewClient(dtP.Url, dtrack.WithAPIKey(dtP.ApiKey), dtrack.WithDebug(false)) - if err != nil { log.Fatalf("Failed to create Dependency-Track client: %s", err) } @@ -81,7 +80,7 @@ func DtrackScore(ctx context.Context, dtP *DtParams) error { ep := &Params{} ep.Path = append(ep.Path, f.Name()) - doc, scores, err := processFile(ctx, ep, ep.Path[0]) + doc, scores, err := processFile(ctx, ep, ep.Path[0], nil) if err != nil { return err } @@ -89,7 +88,7 @@ func DtrackScore(ctx context.Context, dtP *DtParams) error { if dtP.TagProjectWithScore { log.Debugf("Project: %+v", prj.Tags) - //remove old score + // remove old score prj.Tags = lo.Filter(prj.Tags, func(t dtrack.Tag, _ int) bool { return !strings.HasPrefix(t.Name, "sbomqs=") }) diff --git a/pkg/engine/score.go b/pkg/engine/score.go index 2a1e0ca..170d908 100644 --- a/pkg/engine/score.go +++ b/pkg/engine/score.go @@ -17,14 +17,21 @@ package engine import ( "context" "fmt" + "net/url" "os" "path/filepath" "strings" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" "github.com/interlynk-io/sbomqs/pkg/logger" "github.com/interlynk-io/sbomqs/pkg/reporter" "github.com/interlynk-io/sbomqs/pkg/sbom" "github.com/interlynk-io/sbomqs/pkg/scorer" + "github.com/interlynk-io/sbomqs/pkg/source" ) type Params struct { @@ -73,40 +80,117 @@ func handlePaths(ctx context.Context, ep *Params) error { var scores []scorer.Scores for _, path := range ep.Path { - log.Debugf("Processing path :%s\n", path) - pathInfo, _ := os.Stat(path) - if pathInfo.IsDir() { - files, err := os.ReadDir(path) + if source.IsGit(path) { + fmt.Println("Yes, it's a git url: ", path) + + fs := memfs.New() + + gitURL, err := url.Parse(path) if err != nil { - log.Debugf("os.ReadDir failed for path:%s\n", path) - log.Debugf("%s\n", err) - continue + log.Fatalf("err:%v ", err) } - for _, file := range files { - log.Debugf("Processing file :%s\n", file.Name()) - if file.IsDir() { - continue - } - path := filepath.Join(path, file.Name()) - doc, scs, err := processFile(ctx, ep, path) + fmt.Println("parse gitURL: ", gitURL) + + pathElems := strings.Split(gitURL.Path[1:], "/") + if len(pathElems) <= 1 { + log.Fatalf("invalid URL path %s - expected https://github.com/:owner/:repository/:branch (without --git-branch flag) OR https://github.com/:owner/:repository/:directory (with --git-branch flag)", gitURL.Path) + } + + fmt.Println("pathElems: ", pathElems) + fmt.Println("Before gitURL.Path: ", gitURL.Path) + + var gitBranch string + + if strings.Contains(strings.Join(pathElems, " "), "main") { + gitBranch = "main" + } else if strings.Contains(strings.Join(pathElems, " "), "master") { + gitBranch = "master" + } else { + gitBranch = "null" + } + fmt.Println("gitBranch: ", gitBranch) + + gitURL.Path = strings.Join([]string{pathElems[0], pathElems[1]}, "/") + fmt.Println("After gitURL.Path: ", gitURL.Path) + + repoURL := gitURL.String() + fmt.Println("repoURL: ", repoURL) + + fileOrDirPath := strings.Join(pathElems[4:], "/") + fmt.Println("lastPathElement: ", fileOrDirPath) + + cloneOptions := &git.CloneOptions{ + URL: repoURL, + ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", gitBranch)), + Depth: 1, + Progress: os.Stdout, + SingleBranch: true, + } + + _, err = git.Clone(memory.NewStorage(), fs, cloneOptions) + if err != nil { + log.Fatalf("Failed to clone repository: %s", err) + } + + var baths []string + if baths, err = source.ProcessPath(fs, fileOrDirPath); err != nil { + log.Fatalf("Error processing path: %v", err) + } + fmt.Println("baths: ", paths) + + for _, p := range baths { + fmt.Println("File Path:", p) + + doc, scs, err := processFile(ctx, ep, p, fs) if err != nil { continue } + fmt.Println("scs.AvgScore: ", scs.AvgScore()) + docs = append(docs, doc) scores = append(scores, scs) - paths = append(paths, path) + paths = append(paths, p) + fmt.Println("PATHS: ", paths) } - continue - } - doc, scs, err := processFile(ctx, ep, path) - if err != nil { - continue + } else { + + log.Debugf("Processing path :%s\n", path) + pathInfo, _ := os.Stat(path) + if pathInfo.IsDir() { + files, err := os.ReadDir(path) + if err != nil { + log.Debugf("os.ReadDir failed for path:%s\n", path) + log.Debugf("%s\n", err) + continue + } + for _, file := range files { + log.Debugf("Processing file :%s\n", file.Name()) + if file.IsDir() { + continue + } + path := filepath.Join(path, file.Name()) + doc, scs, err := processFile(ctx, ep, path, nil) + if err != nil { + continue + } + docs = append(docs, doc) + scores = append(scores, scs) + paths = append(paths, path) + } + continue + } + + doc, scs, err := processFile(ctx, ep, path, nil) + if err != nil { + continue + } + docs = append(docs, doc) + scores = append(scores, scs) + paths = append(paths, path) } - docs = append(docs, doc) - scores = append(scores, scs) - paths = append(paths, path) } + fmt.Println("Outside for loop and git condition") reportFormat := "detailed" if ep.Basic { @@ -126,30 +210,54 @@ func handlePaths(ctx context.Context, ep *Params) error { return nil } -func processFile(ctx context.Context, ep *Params, path string) (sbom.Document, scorer.Scores, error) { +func processFile(ctx context.Context, ep *Params, path string, fs billy.Filesystem) (sbom.Document, scorer.Scores, error) { + fmt.Println("Inside processFile function") + defer fmt.Println("Exit processFile function") log := logger.FromContext(ctx) log.Debugf("Processing file :%s\n", path) + var doc sbom.Document - if _, err := os.Stat(path); err != nil { - log.Debugf("os.Stat failed for file :%s\n", path) - fmt.Printf("failed to stat %s\n", path) - return nil, nil, err - } + if fs != nil { + fmt.Println("Yes fs contains files") + f, err := fs.Open(path) + if err != nil { + log.Debugf("os.Open failed for file :%s\n", path) + fmt.Printf("failed to open %s\n", path) + return nil, nil, err + } + defer f.Close() - f, err := os.Open(path) - if err != nil { - log.Debugf("os.Open failed for file :%s\n", path) - fmt.Printf("failed to open %s\n", path) - return nil, nil, err - } - defer f.Close() - - doc, err := sbom.NewSBOMDocument(ctx, f) - if err != nil { - log.Debugf("failed to create sbom document for :%s\n", path) - log.Debugf("%s\n", err) - fmt.Printf("failed to parse %s : %s\n", path, err) - return nil, nil, err + doc, err = sbom.NewSBOMDocument(ctx, f) + if err != nil { + log.Debugf("failed to create sbom document for :%s\n", path) + log.Debugf("%s\n", err) + fmt.Printf("failed to parse %s : %s\n", path, err) + return nil, nil, err + } + + } else { + + if _, err := os.Stat(path); err != nil { + log.Debugf("os.Stat failed for file :%s\n", path) + fmt.Printf("failed to stat %s\n", path) + return nil, nil, err + } + + f, err := os.Open(path) + if err != nil { + log.Debugf("os.Open failed for file :%s\n", path) + fmt.Printf("failed to open %s\n", path) + return nil, nil, err + } + defer f.Close() + + doc, err = sbom.NewSBOMDocument(ctx, f) + if err != nil { + log.Debugf("failed to create sbom document for :%s\n", path) + log.Debugf("%s\n", err) + fmt.Printf("failed to parse %s : %s\n", path, err) + return nil, nil, err + } } sr := scorer.NewScorer(ctx, doc) diff --git a/pkg/engine/share.go b/pkg/engine/share.go index 90c6649..e3cacb3 100644 --- a/pkg/engine/share.go +++ b/pkg/engine/share.go @@ -34,13 +34,12 @@ func ShareRun(ctx context.Context, ep *Params) error { log.Fatal("path is required") } - doc, scores, err := processFile(ctx, ep, ep.Path[0]) + doc, scores, err := processFile(ctx, ep, ep.Path[0], nil) if err != nil { return err } url, err := share.Share(ctx, doc, scores, ep.Path[0]) - if err != nil { fmt.Printf("Error sharing file %s: %s", ep.Path, err) return err From 34fe68e4b2ed69c417073d07d906df5ce24ed366 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Fri, 12 Jul 2024 16:28:52 +0530 Subject: [PATCH 02/11] support git urls for file Signed-off-by: Vivek Kumar Sahu --- pkg/engine/score.go | 136 ++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/pkg/engine/score.go b/pkg/engine/score.go index 170d908..f4435b1 100644 --- a/pkg/engine/score.go +++ b/pkg/engine/score.go @@ -17,21 +17,21 @@ package engine import ( "context" "fmt" + "io" + "log" + "net/http" "net/url" "os" "path/filepath" + "regexp" "strings" "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/storage/memory" "github.com/interlynk-io/sbomqs/pkg/logger" "github.com/interlynk-io/sbomqs/pkg/reporter" "github.com/interlynk-io/sbomqs/pkg/sbom" "github.com/interlynk-io/sbomqs/pkg/scorer" - "github.com/interlynk-io/sbomqs/pkg/source" + "github.com/spf13/afero" ) type Params struct { @@ -71,6 +71,40 @@ func Run(ctx context.Context, ep *Params) error { return handlePaths(ctx, ep) } +func handleURL(path string) (string, string, error) { + u, err := url.Parse(path) + if err != nil { + log.Fatalf("Failed to parse urlPath: %v", err) + } + + parts := strings.Split(u.Path, "/") + if len(parts) < 5 { + log.Fatalf("invalid GitHub URL: %v", path) + } + + if len(parts) < 5 { + log.Fatalf("invalid GitHub URL: %v", path) + } + fmt.Println("Parts: ", parts) + + sbomFilePath := strings.Join(parts[5:], "/") + fmt.Println("sbomFilePath: ", sbomFilePath) + + rawURL := strings.Replace(path, "github.com", "raw.githubusercontent.com", 1) + rawURL = strings.Replace(rawURL, "/blob/", "/", 1) + fmt.Println("rawURL: ", rawURL) + + return sbomFilePath, rawURL, err +} + +func IsURL(in string) bool { + return regexp.MustCompile("^(http|https)://").MatchString(in) +} + +func IsGit(in string) bool { + return regexp.MustCompile("^(http|https)://github.com").MatchString(in) +} + func handlePaths(ctx context.Context, ep *Params) error { log := logger.FromContext(ctx) log.Debug("engine.handlePaths()") @@ -80,79 +114,49 @@ func handlePaths(ctx context.Context, ep *Params) error { var scores []scorer.Scores for _, path := range ep.Path { - if source.IsGit(path) { - fmt.Println("Yes, it's a git url: ", path) - - fs := memfs.New() - - gitURL, err := url.Parse(path) - if err != nil { - log.Fatalf("err:%v ", err) + if IsURL(path) { + if IsGit(path) { + return fmt.Errorf("path is not a git URL: %s", path) } - fmt.Println("parse gitURL: ", gitURL) - - pathElems := strings.Split(gitURL.Path[1:], "/") - if len(pathElems) <= 1 { - log.Fatalf("invalid URL path %s - expected https://github.com/:owner/:repository/:branch (without --git-branch flag) OR https://github.com/:owner/:repository/:directory (with --git-branch flag)", gitURL.Path) + fmt.Println("It's a GitHub URL: ", path) + sbomFilePath, rawURL, err := handleURL(path) + if err != nil { + log.Fatal("failed to get sbomFilePath, rawURL: %w", err) } - fmt.Println("pathElems: ", pathElems) - fmt.Println("Before gitURL.Path: ", gitURL.Path) - - var gitBranch string + fs := afero.NewMemMapFs() - if strings.Contains(strings.Join(pathElems, " "), "main") { - gitBranch = "main" - } else if strings.Contains(strings.Join(pathElems, " "), "master") { - gitBranch = "master" - } else { - gitBranch = "null" + file, err := fs.Create(sbomFilePath) + if err != nil { + return err } - fmt.Println("gitBranch: ", gitBranch) - - gitURL.Path = strings.Join([]string{pathElems[0], pathElems[1]}, "/") - fmt.Println("After gitURL.Path: ", gitURL.Path) - repoURL := gitURL.String() - fmt.Println("repoURL: ", repoURL) - - fileOrDirPath := strings.Join(pathElems[4:], "/") - fmt.Println("lastPathElement: ", fileOrDirPath) - - cloneOptions := &git.CloneOptions{ - URL: repoURL, - ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", gitBranch)), - Depth: 1, - Progress: os.Stdout, - SingleBranch: true, + resp, err := http.Get(rawURL) + if err != nil { + log.Fatalf("failed to get data: %v", err) } + defer resp.Body.Close() - _, err = git.Clone(memory.NewStorage(), fs, cloneOptions) + // Ensure the response is OK + if resp.StatusCode != http.StatusOK { + log.Fatalf("failed to download file: %s", resp.Status) + } + _, err = io.Copy(file, resp.Body) if err != nil { - log.Fatalf("Failed to clone repository: %s", err) + log.Fatalf("failed to copy in file: %w", err) } - var baths []string - if baths, err = source.ProcessPath(fs, fileOrDirPath); err != nil { - log.Fatalf("Error processing path: %v", err) + doc, err := sbom.NewSBOMDocument(ctx, file) + if err != nil { + log.Fatalf("failed to parse SBOM document: %w", err) } - fmt.Println("baths: ", paths) - - for _, p := range baths { - fmt.Println("File Path:", p) - - doc, scs, err := processFile(ctx, ep, p, fs) - if err != nil { - continue - } - fmt.Println("scs.AvgScore: ", scs.AvgScore()) - docs = append(docs, doc) - scores = append(scores, scs) - paths = append(paths, p) - fmt.Println("PATHS: ", paths) - } + sr := scorer.NewScorer(ctx, doc) + score := sr.Score() + docs = append(docs, doc) + scores = append(scores, score) + paths = append(paths, sbomFilePath) } else { log.Debugf("Processing path :%s\n", path) @@ -190,7 +194,6 @@ func handlePaths(ctx context.Context, ep *Params) error { paths = append(paths, path) } } - fmt.Println("Outside for loop and git condition") reportFormat := "detailed" if ep.Basic { @@ -211,14 +214,11 @@ func handlePaths(ctx context.Context, ep *Params) error { } func processFile(ctx context.Context, ep *Params, path string, fs billy.Filesystem) (sbom.Document, scorer.Scores, error) { - fmt.Println("Inside processFile function") - defer fmt.Println("Exit processFile function") log := logger.FromContext(ctx) log.Debugf("Processing file :%s\n", path) var doc sbom.Document if fs != nil { - fmt.Println("Yes fs contains files") f, err := fs.Open(path) if err != nil { log.Debugf("os.Open failed for file :%s\n", path) From 528543196784b80ca80e89de1eb92497c82f4c82 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Fri, 12 Jul 2024 16:34:04 +0530 Subject: [PATCH 03/11] fix lint issue Signed-off-by: Vivek Kumar Sahu --- cmd/score.go | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/cmd/score.go b/cmd/score.go index c23418f..df2ec34 100644 --- a/cmd/score.go +++ b/cmd/score.go @@ -37,25 +37,25 @@ var ( ) type userCmd struct { - //input control + // input control path []string - //filter control + // filter control category string features []string - //output control + // output control json bool basic bool detailed bool - //directory control + // directory control recurse bool - //debug control + // debug control debug bool - //config control + // config control configPath string } @@ -69,7 +69,6 @@ var scoreCmd = &cobra.Command{ if len(inFile) <= 0 && len(inDirPath) <= 0 { return fmt.Errorf("provide a path to an sbom file or directory of sbom files") } - } return nil }, @@ -94,10 +93,11 @@ func processScore(cmd *cobra.Command, args []string) error { engParams := toEngineParams(uCmd) return engine.Run(ctx, engParams) } + func toUserCmd(cmd *cobra.Command, args []string) *userCmd { uCmd := &userCmd{} - //input control + // input control if len(args) <= 0 { if len(inFile) > 0 { uCmd.path = append(uCmd.path, inFile) @@ -110,13 +110,13 @@ func toUserCmd(cmd *cobra.Command, args []string) *userCmd { uCmd.path = append(uCmd.path, args[0:]...) } - //config control + // config control if configPath == "" { uCmd.configPath, _ = cmd.Flags().GetString("configpath") } else { uCmd.configPath = configPath } - //filter control + // filter control if category == "" { uCmd.category, _ = cmd.Flags().GetString("category") } else { @@ -128,7 +128,7 @@ func toUserCmd(cmd *cobra.Command, args []string) *userCmd { uCmd.features = strings.Split(f, ",") } - //output control + // output control uCmd.json, _ = cmd.Flags().GetBool("json") uCmd.basic, _ = cmd.Flags().GetBool("basic") uCmd.detailed, _ = cmd.Flags().GetBool("detailed") @@ -139,7 +139,7 @@ func toUserCmd(cmd *cobra.Command, args []string) *userCmd { uCmd.detailed = strings.ToLower(reportFormat) == "detailed" } - //debug control + // debug control uCmd.debug, _ = cmd.Flags().GetBool("debug") return uCmd @@ -165,14 +165,8 @@ func validatePath(path string) error { } return nil } -func validateFlags(cmd *userCmd) error { - - for _, path := range cmd.path { - if err := validatePath(path); err != nil { - return fmt.Errorf("invalid path: %w", err) - } - } +func validateFlags(cmd *userCmd) error { if cmd.configPath != "" { if err := validatePath(cmd.configPath); err != nil { return fmt.Errorf("invalid config path: %w", err) @@ -185,36 +179,37 @@ func validateFlags(cmd *userCmd) error { return nil } + func init() { rootCmd.AddCommand(scoreCmd) - //Config Control + // Config Control scoreCmd.Flags().StringP("configpath", "", "", "scoring based on config path") - //Filter Control + // Filter Control scoreCmd.Flags().StringP("category", "c", "", "filter by category") scoreCmd.Flags().StringP("feature", "f", "", "filter by feature") - //Spec Control + // Spec Control scoreCmd.Flags().BoolP("spdx", "", false, "limit scoring to spdx sboms") scoreCmd.Flags().BoolP("cdx", "", false, "limit scoring to cdx sboms") scoreCmd.MarkFlagsMutuallyExclusive("spdx", "cdx") scoreCmd.Flags().MarkHidden("spdx") scoreCmd.Flags().MarkHidden("cdx") - //Directory Control + // Directory Control scoreCmd.Flags().BoolP("recurse", "r", false, "recurse into subdirectories") scoreCmd.Flags().MarkHidden("recurse") - //Output Control + // Output Control scoreCmd.Flags().BoolP("json", "j", false, "results in json") scoreCmd.Flags().BoolP("detailed", "d", false, "results in table format, default") scoreCmd.Flags().BoolP("basic", "b", false, "results in single line format") - //Debug Control + // Debug Control scoreCmd.Flags().BoolP("debug", "D", false, "enable debug logging") - //Deprecated + // Deprecated scoreCmd.Flags().StringVar(&inFile, "filepath", "", "sbom file path") scoreCmd.Flags().StringVar(&inDirPath, "dirpath", "", "sbom dir path") scoreCmd.MarkFlagsMutuallyExclusive("filepath", "dirpath") From 57cb807b0ed947cce8de54598162a5b186e330c9 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Fri, 12 Jul 2024 16:34:48 +0530 Subject: [PATCH 04/11] handle non git path Signed-off-by: Vivek Kumar Sahu --- go.mod | 3 ++- go.sum | 8 ++++++-- pkg/engine/score.go | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 31843a2..72a9f2b 100644 --- a/go.mod +++ b/go.mod @@ -32,12 +32,13 @@ require ( github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect github.com/cloudflare/circl v1.3.9 // indirect github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect + github.com/go-git/go-billy/v5 v5.5.0 github.com/google/go-querystring v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/spdx/gordf v0.0.0-20221230105357-b735bd5aac89 // indirect + github.com/spf13/afero v1.11.0 github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.24.0 // indirect diff --git a/go.sum b/go.sum index 59426d9..86d56eb 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/github/go-spdx/v2 v2.3.1 h1:ffGuHTbHuHzWPt53n8f9o8clGutuLPObo3zB4JAjxU8= github.com/github/go-spdx/v2 v2.3.1/go.mod h1:2ZxKsOhvBp+OYBDlsGnUMcchLeo2mrpEBn2L1C+U3IQ= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -59,8 +61,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ= github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= @@ -71,6 +73,8 @@ github.com/spdx/gordf v0.0.0-20221230105357-b735bd5aac89 h1:dArkMwZ7Mf2JiU8Ofdmq github.com/spdx/gordf v0.0.0-20221230105357-b735bd5aac89/go.mod h1:uKWaldnbMnjsSAXRurWqqrdyZen1R7kxl8TkmWk2OyM= github.com/spdx/tools-golang v0.5.5 h1:61c0KLfAcNqAjlg6UNMdkwpMernhw3zVRwDZ2x9XOmk= github.com/spdx/tools-golang v0.5.5/go.mod h1:MVIsXx8ZZzaRWNQpUDhC4Dud34edUYJYecciXgrw5vE= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/pkg/engine/score.go b/pkg/engine/score.go index f4435b1..4c91aa1 100644 --- a/pkg/engine/score.go +++ b/pkg/engine/score.go @@ -115,7 +115,7 @@ func handlePaths(ctx context.Context, ep *Params) error { for _, path := range ep.Path { if IsURL(path) { - if IsGit(path) { + if !IsGit(path) { return fmt.Errorf("path is not a git URL: %s", path) } fmt.Println("It's a GitHub URL: ", path) From 2a10f08604c98e5c662f4c91a5695dd77b58bf01 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Wed, 17 Jul 2024 14:35:22 +0530 Subject: [PATCH 05/11] add support to handle more url Signed-off-by: Vivek Kumar Sahu --- pkg/engine/score.go | 109 ++++++++++++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 35 deletions(-) diff --git a/pkg/engine/score.go b/pkg/engine/score.go index 4c91aa1..17b8ff5 100644 --- a/pkg/engine/score.go +++ b/pkg/engine/score.go @@ -115,48 +115,87 @@ func handlePaths(ctx context.Context, ep *Params) error { for _, path := range ep.Path { if IsURL(path) { - if !IsGit(path) { - return fmt.Errorf("path is not a git URL: %s", path) - } - fmt.Println("It's a GitHub URL: ", path) - sbomFilePath, rawURL, err := handleURL(path) - if err != nil { - log.Fatal("failed to get sbomFilePath, rawURL: %w", err) - } + if IsGit(path) { + log.Debugf("Processing Git URL path :%s\n", path) - fs := afero.NewMemMapFs() + sbomFilePath, rawURL, err := handleURL(path) + if err != nil { + log.Fatal("failed to get sbomFilePath, rawURL: %w", err) + } - file, err := fs.Create(sbomFilePath) - if err != nil { - return err - } + fs := afero.NewMemMapFs() - resp, err := http.Get(rawURL) - if err != nil { - log.Fatalf("failed to get data: %v", err) - } - defer resp.Body.Close() + file, err := fs.Create(sbomFilePath) + if err != nil { + return err + } - // Ensure the response is OK - if resp.StatusCode != http.StatusOK { - log.Fatalf("failed to download file: %s", resp.Status) - } - _, err = io.Copy(file, resp.Body) - if err != nil { - log.Fatalf("failed to copy in file: %w", err) - } + resp, err := http.Get(rawURL) + if err != nil { + log.Fatalf("failed to get data: %v", err) + } + defer resp.Body.Close() - doc, err := sbom.NewSBOMDocument(ctx, file) - if err != nil { - log.Fatalf("failed to parse SBOM document: %w", err) - } + // Ensure the response is OK + if resp.StatusCode != http.StatusOK { + log.Fatalf("failed to download file: %s", resp.Status) + } + _, err = io.Copy(file, resp.Body) + if err != nil { + log.Fatalf("failed to copy in file: %w", err) + } + + doc, err := sbom.NewSBOMDocument(ctx, file) + if err != nil { + log.Fatalf("failed to parse SBOM document: %w", err) + } - sr := scorer.NewScorer(ctx, doc) - score := sr.Score() + sr := scorer.NewScorer(ctx, doc) + score := sr.Score() - docs = append(docs, doc) - scores = append(scores, score) - paths = append(paths, sbomFilePath) + docs = append(docs, doc) + scores = append(scores, score) + paths = append(paths, sbomFilePath) + + } else { + log.Debugf("Processing URL path :%s\n", path) + filename := path + fs := afero.NewMemMapFs() + + file, err := fs.Create(filename) + if err != nil { + log.Fatal("failed to get create file: %w", err) + } + defer file.Close() + + resp, err := http.Get(path) + if err != nil { + panic(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + panic("Bad response from server: " + resp.Status) + } + + _, err = io.Copy(file, resp.Body) + if err != nil { + log.Fatalf("failed to copy file: %w", err) + } + + doc, err := sbom.NewSBOMDocument(ctx, file) + if err != nil { + log.Fatalf("failed to parse SBOM document: %w", err) + } + + sr := scorer.NewScorer(ctx, doc) + score := sr.Score() + + docs = append(docs, doc) + scores = append(scores, score) + paths = append(paths, path) + + } } else { log.Debugf("Processing path :%s\n", path) From 8adfafc6a85a35a321827d74126eb52c6231b995 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Thu, 18 Jul 2024 11:38:04 +0530 Subject: [PATCH 06/11] function to process url Signed-off-by: Vivek Kumar Sahu --- pkg/engine/score.go | 115 +++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 71 deletions(-) diff --git a/pkg/engine/score.go b/pkg/engine/score.go index 17b8ff5..a06b5fd 100644 --- a/pkg/engine/score.go +++ b/pkg/engine/score.go @@ -105,6 +105,25 @@ func IsGit(in string) bool { return regexp.MustCompile("^(http|https)://github.com").MatchString(in) } +func ProcessURL(url string, file afero.File) (afero.File, error) { + resp, err := http.Get(url) + if err != nil { + log.Fatalf("failed to get data: %v", err) + } + defer resp.Body.Close() + + // Ensure the response is OK + if resp.StatusCode != http.StatusOK { + log.Fatalf("failed to download file: %s", resp.Status) + } + _, err = io.Copy(file, resp.Body) + if err != nil { + log.Fatalf("failed to copy in file: %w", err) + } + + return file, err +} + func handlePaths(ctx context.Context, ep *Params) error { log := logger.FromContext(ctx) log.Debug("engine.handlePaths()") @@ -115,87 +134,41 @@ func handlePaths(ctx context.Context, ep *Params) error { for _, path := range ep.Path { if IsURL(path) { - if IsGit(path) { - log.Debugf("Processing Git URL path :%s\n", path) - - sbomFilePath, rawURL, err := handleURL(path) - if err != nil { - log.Fatal("failed to get sbomFilePath, rawURL: %w", err) - } - - fs := afero.NewMemMapFs() - - file, err := fs.Create(sbomFilePath) - if err != nil { - return err - } - - resp, err := http.Get(rawURL) - if err != nil { - log.Fatalf("failed to get data: %v", err) - } - defer resp.Body.Close() + log.Debugf("Processing Git URL path :%s\n", path) - // Ensure the response is OK - if resp.StatusCode != http.StatusOK { - log.Fatalf("failed to download file: %s", resp.Status) - } - _, err = io.Copy(file, resp.Body) - if err != nil { - log.Fatalf("failed to copy in file: %w", err) - } + url, sbomFilePath := path, path + var err error - doc, err := sbom.NewSBOMDocument(ctx, file) + if IsGit(url) { + sbomFilePath, url, err = handleURL(path) if err != nil { - log.Fatalf("failed to parse SBOM document: %w", err) - } - - sr := scorer.NewScorer(ctx, doc) - score := sr.Score() - - docs = append(docs, doc) - scores = append(scores, score) - paths = append(paths, sbomFilePath) - - } else { - log.Debugf("Processing URL path :%s\n", path) - filename := path - fs := afero.NewMemMapFs() - - file, err := fs.Create(filename) - if err != nil { - log.Fatal("failed to get create file: %w", err) - } - defer file.Close() - - resp, err := http.Get(path) - if err != nil { - panic(err) + log.Fatal("failed to get sbomFilePath, rawURL: %w", err) } - defer resp.Body.Close() + } + fs := afero.NewMemMapFs() - if resp.StatusCode != http.StatusOK { - panic("Bad response from server: " + resp.Status) - } + file, err := fs.Create(sbomFilePath) + if err != nil { + return err + } - _, err = io.Copy(file, resp.Body) - if err != nil { - log.Fatalf("failed to copy file: %w", err) - } + f, err := ProcessURL(url, file) + if err != nil { + return err + } - doc, err := sbom.NewSBOMDocument(ctx, file) - if err != nil { - log.Fatalf("failed to parse SBOM document: %w", err) - } + doc, err := sbom.NewSBOMDocument(ctx, f) + if err != nil { + log.Fatalf("failed to parse SBOM document: %w", err) + } - sr := scorer.NewScorer(ctx, doc) - score := sr.Score() + sr := scorer.NewScorer(ctx, doc) + score := sr.Score() - docs = append(docs, doc) - scores = append(scores, score) - paths = append(paths, path) + docs = append(docs, doc) + scores = append(scores, score) + paths = append(paths, sbomFilePath) - } } else { log.Debugf("Processing path :%s\n", path) From 8c5023a3ac1688a1f9fb4c756008445f6b6a75d1 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Thu, 18 Jul 2024 12:08:52 +0530 Subject: [PATCH 07/11] add url support for compliance cmd Signed-off-by: Vivek Kumar Sahu --- pkg/engine/compliance.go | 76 +++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/pkg/engine/compliance.go b/pkg/engine/compliance.go index b187785..ee28424 100644 --- a/pkg/engine/compliance.go +++ b/pkg/engine/compliance.go @@ -22,6 +22,7 @@ import ( "github.com/interlynk-io/sbomqs/pkg/compliance" "github.com/interlynk-io/sbomqs/pkg/logger" "github.com/interlynk-io/sbomqs/pkg/sbom" + "github.com/spf13/afero" ) func ComplianceRun(ctx context.Context, ep *Params) error { @@ -74,27 +75,60 @@ func getSbomDocument(ctx context.Context, ep *Params) (*sbom.Document, error) { log.Debugf("engine.getSbomDocument()") path := ep.Path[0] - - if _, err := os.Stat(path); err != nil { - log.Debugf("os.Stat failed for file :%s\n", path) - fmt.Printf("failed to stat %s\n", path) - return nil, err - } - - f, err := os.Open(path) - if err != nil { - log.Debugf("os.Open failed for file :%s\n", path) - fmt.Printf("failed to open %s\n", path) - return nil, err - } - defer f.Close() - - doc, err := sbom.NewSBOMDocument(ctx, f) - if err != nil { - log.Debugf("failed to create sbom document for :%s\n", path) - log.Debugf("%s\n", err) - fmt.Printf("failed to parse %s : %s\n", path, err) - return nil, err + var doc sbom.Document + + if IsURL(path) { + log.Debugf("Processing Git URL path :%s\n", path) + + url, sbomFilePath := path, path + var err error + + if IsGit(url) { + sbomFilePath, url, err = handleURL(path) + if err != nil { + log.Fatal("failed to get sbomFilePath, rawURL: %w", err) + } + } + fs := afero.NewMemMapFs() + + file, err := fs.Create(sbomFilePath) + if err != nil { + return nil, err + } + + f, err := ProcessURL(url, file) + if err != nil { + return nil, err + } + + doc, err = sbom.NewSBOMDocument(ctx, f) + if err != nil { + log.Fatalf("failed to parse SBOM document: %w", err) + } + + } else { + + if _, err := os.Stat(path); err != nil { + log.Debugf("os.Stat failed for file :%s\n", path) + fmt.Printf("failed to stat %s\n", path) + return nil, err + } + + f, err := os.Open(path) + if err != nil { + log.Debugf("os.Open failed for file :%s\n", path) + fmt.Printf("failed to open %s\n", path) + return nil, err + } + defer f.Close() + + doc, err = sbom.NewSBOMDocument(ctx, f) + if err != nil { + log.Debugf("failed to create sbom document for :%s\n", path) + log.Debugf("%s\n", err) + fmt.Printf("failed to parse %s : %s\n", path, err) + return nil, err + } } return &doc, nil From 22be5f9b0067717bc791263254f941a26cbe5e45 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Thu, 18 Jul 2024 12:09:10 +0530 Subject: [PATCH 08/11] handle empty files Signed-off-by: Vivek Kumar Sahu --- pkg/engine/score.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/engine/score.go b/pkg/engine/score.go index a06b5fd..068989d 100644 --- a/pkg/engine/score.go +++ b/pkg/engine/score.go @@ -108,17 +108,27 @@ func IsGit(in string) bool { func ProcessURL(url string, file afero.File) (afero.File, error) { resp, err := http.Get(url) if err != nil { - log.Fatalf("failed to get data: %v", err) + fmt.Println("FAiled to get http URL") + return nil, fmt.Errorf("failed to get data: %v", err) } defer resp.Body.Close() // Ensure the response is OK if resp.StatusCode != http.StatusOK { - log.Fatalf("failed to download file: %s", resp.Status) + return nil, fmt.Errorf("failed to download file: %s", resp.Status) } _, err = io.Copy(file, resp.Body) if err != nil { - log.Fatalf("failed to copy in file: %w", err) + return nil, fmt.Errorf("failed to copy in file: %w", err) + } + + // Check if the file is empty + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("failed to stat file: %w", err) + } + if info.Size() == 0 { + return nil, fmt.Errorf("downloaded file is empty") } return file, err From 580a9cc0928d3d2648bd64143dbeb6e487ddf736 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Thu, 18 Jul 2024 12:20:10 +0530 Subject: [PATCH 09/11] remove print stmt Signed-off-by: Vivek Kumar Sahu --- pkg/engine/score.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/engine/score.go b/pkg/engine/score.go index 068989d..79b541d 100644 --- a/pkg/engine/score.go +++ b/pkg/engine/score.go @@ -85,14 +85,11 @@ func handleURL(path string) (string, string, error) { if len(parts) < 5 { log.Fatalf("invalid GitHub URL: %v", path) } - fmt.Println("Parts: ", parts) sbomFilePath := strings.Join(parts[5:], "/") - fmt.Println("sbomFilePath: ", sbomFilePath) rawURL := strings.Replace(path, "github.com", "raw.githubusercontent.com", 1) rawURL = strings.Replace(rawURL, "/blob/", "/", 1) - fmt.Println("rawURL: ", rawURL) return sbomFilePath, rawURL, err } @@ -108,7 +105,6 @@ func IsGit(in string) bool { func ProcessURL(url string, file afero.File) (afero.File, error) { resp, err := http.Get(url) if err != nil { - fmt.Println("FAiled to get http URL") return nil, fmt.Errorf("failed to get data: %v", err) } defer resp.Body.Close() From 522774f562364047072796fbc541170b8d09e0b5 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Thu, 18 Jul 2024 16:55:16 +0530 Subject: [PATCH 10/11] fix url containing slash Signed-off-by: Vivek Kumar Sahu --- pkg/engine/score.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/engine/score.go b/pkg/engine/score.go index 79b541d..67ffac3 100644 --- a/pkg/engine/score.go +++ b/pkg/engine/score.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "log" "net/http" "net/url" "os" @@ -74,20 +73,25 @@ func Run(ctx context.Context, ep *Params) error { func handleURL(path string) (string, string, error) { u, err := url.Parse(path) if err != nil { - log.Fatalf("Failed to parse urlPath: %v", err) + return "", "", fmt.Errorf("failed to parse urlPath: %v", err) } parts := strings.Split(u.Path, "/") - if len(parts) < 5 { - log.Fatalf("invalid GitHub URL: %v", path) - } + containSlash := strings.HasSuffix(u.Path, "/") + var sbomFilePath string - if len(parts) < 5 { - log.Fatalf("invalid GitHub URL: %v", path) + if containSlash { + if len(parts) < 7 { + return "", "", fmt.Errorf("invalid GitHub URL: %v", path) + } + sbomFilePath = strings.Join(parts[5:len(parts)-1], "/") + } else { + if len(parts) < 6 { + return "", "", fmt.Errorf("invalid GitHub URL: %v", path) + } + sbomFilePath = strings.Join(parts[5:], "/") } - sbomFilePath := strings.Join(parts[5:], "/") - rawURL := strings.Replace(path, "github.com", "raw.githubusercontent.com", 1) rawURL = strings.Replace(rawURL, "/blob/", "/", 1) @@ -124,7 +128,7 @@ func ProcessURL(url string, file afero.File) (afero.File, error) { return nil, fmt.Errorf("failed to stat file: %w", err) } if info.Size() == 0 { - return nil, fmt.Errorf("downloaded file is empty") + return nil, fmt.Errorf("downloaded file is empty: %v", info.Size()) } return file, err From bbeed144d6d8a7548a4c1678ff382b580af1ed21 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Thu, 18 Jul 2024 16:55:35 +0530 Subject: [PATCH 11/11] add test Signed-off-by: Vivek Kumar Sahu --- go.mod | 4 ++ pkg/engine/score_test.go | 108 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 pkg/engine/score_test.go diff --git a/go.mod b/go.mod index 72a9f2b..c342cfc 100644 --- a/go.mod +++ b/go.mod @@ -23,8 +23,12 @@ require ( ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/pkg/engine/score_test.go b/pkg/engine/score_test.go new file mode 100644 index 0000000..ee61771 --- /dev/null +++ b/pkg/engine/score_test.go @@ -0,0 +1,108 @@ +package engine + +import ( + "fmt" + "log" + "net/http" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" +) + +func TestHandleURL(t *testing.T) { + testCases := []struct { + name string + input string + expectedPath string + expectedRawURL string + expectedError bool + }{ + { + name: "Valid URL", + input: "https://github.com/interlynk-io/sbomqs/blob/main/samples/sbomqs-spdx-syft.json", + expectedPath: "samples/sbomqs-spdx-syft.json", + expectedRawURL: "https://raw.githubusercontent.com/interlynk-io/sbomqs/main/samples/sbomqs-spdx-syft.json", + expectedError: false, + }, + { + name: "Valid URL with direct file", + input: "https://github.com/viveksahu26/go-url/blob/main/spdx.json", + expectedError: false, + expectedPath: "spdx.json", + expectedRawURL: "https://raw.githubusercontent.com/viveksahu26/go-url/main/spdx.json", + }, + { + name: "Invalid URL with not enough parts", + input: "https://github.com/interlynk-io/sbomqs/blob/main/", + expectedPath: "", + expectedRawURL: "", + expectedError: true, + }, + { + name: "Malformed URL", + input: "invalid-url", + expectedPath: "", + expectedRawURL: "", + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + sbomFilePath, rawURL, err := handleURL(tc.input) + if tc.expectedError { + assert.Error(t, err) + assert.Equal(t, tc.expectedPath, sbomFilePath) + assert.Equal(t, tc.expectedRawURL, rawURL) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.expectedPath, sbomFilePath) + assert.Equal(t, tc.expectedRawURL, rawURL) + } + }) + } +} + +// TestProcessURL function +func TestProcessURL(t *testing.T) { + tests := []struct { + name string + url string + statusCode int + expectedError bool + expectedErrorMessage error + }{ + { + name: "Successful download", + url: "https://github.com/interlynk-io/sbomqs/blob/main/samples/sbomqs-spdx-syft.json", + statusCode: http.StatusOK, + expectedError: false, + expectedErrorMessage: nil, + }, + { + name: "Failed to get data", + url: "http://example.com/file.txt", + statusCode: http.StatusNotFound, + expectedError: true, + expectedErrorMessage: fmt.Errorf("failed to download file: %s %s", "404", http.StatusText(http.StatusNotFound)), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + file, err := fs.Create("testfile.txt") + if err != nil { + log.Fatalf("error: %v", err) + } + + _, err = ProcessURL(tt.url, file) + if tt.expectedError { + assert.EqualError(t, err, tt.expectedErrorMessage.Error()) + } else { + assert.NoError(t, err) + } + }) + } +}