Better progress logging

Signed-off-by: Christian Dupuis <[email protected]>

Author: cdupuis

commands/cmd.go

@@ -55,6 +55,8 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 		cli.DisableFlagsInUseLine(cmd)
 	}
 
+	skill.Log.SetOutput(os.Stderr)
+
 	config := dockerCli.ConfigFile()
 
 	var (
@@ -85,7 +87,7 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 				return err
 			}
 			if valid, err := query.CheckAuth(workspace, apiKey); err == nil && valid {
-				skill.Log.Info("Login successful")
+				fmt.Println("Login successful")
 				config.SetPluginConfig("index", "workspace", workspace)
 				config.SetPluginConfig("index", "api-key", apiKey)
 				return config.Save()
@@ -130,7 +132,7 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 				_ = os.WriteFile(output, js, 0644)
 				skill.Log.Infof("SBOM written to %s", output)
 			} else {
-				os.Stdout.WriteString(string(js) + "\n")
+				fmt.Println(string(js))
 			}
 			return nil
 		},
@@ -214,27 +216,27 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 
 			if len(*cves) > 0 {
 				for _, c := range *cves {
-					skill.Log.Warnf("Detected %s at", cve)
-					skill.Log.Warnf("")
+					fmt.Println(fmt.Sprintf("Detected %s at", cve))
+					fmt.Println("")
 					purl := c.Purl
 					for _, p := range sb.Artifacts {
 						if p.Purl == purl {
-							skill.Log.Warnf("  %s", p.Purl)
+							fmt.Println(fmt.Sprintf("  %s", p.Purl))
 							loc := p.Locations[0]
 							for i, l := range sb.Source.Image.Config.RootFS.DiffIDs {
 								if l.String() == loc.DiffId {
 									h := sb.Source.Image.Config.History[i]
-									skill.Log.Warnf("    ")
-									skill.Log.Warnf("    Instruction: %s", h.CreatedBy)
-									skill.Log.Warnf("    Layer %d: %s", i, loc.Digest)
+									fmt.Println("    ")
+									fmt.Println(fmt.Sprintf("    Instruction: %s", h.CreatedBy))
+									fmt.Println(fmt.Sprintf("    Layer %d: %s", i, loc.Digest))
 								}
 							}
 						}
 					}
 				}
 				os.Exit(1)
 			} else {
-				skill.Log.Infof("%s not detected", cve)
+				fmt.Println(fmt.Sprintf("%s not detected", cve))
 				os.Exit(0)
 			}
 			return nil

registry/save.go

@@ -24,14 +24,15 @@ import (
 
 	"github.com/atomist-skills/go-skill"
 	"github.com/docker/docker/client"
+	"github.com/dustin/go-humanize"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
-	"github.com/google/go-containerregistry/pkg/v1/empty"
-	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 type ImageId struct {
@@ -58,11 +59,13 @@ func (i ImageId) String() string {
 	return i.name
 }
 
+type Cleanup = func()
+
 // SaveImage stores the v1.Image at path returned in OCI format
-func SaveImage(image string, client client.APIClient) (v1.Image, string, error) {
+func SaveImage(image string, client client.APIClient) (v1.Image, string, Cleanup, error) {
 	ref, err := name.ParseReference(image)
 	if err != nil {
-		return nil, "", errors.Wrapf(err, "failed to parse reference: %s", image)
+		return nil, "", nil, errors.Wrapf(err, "failed to parse reference: %s", image)
 	}
 
 	var path string
@@ -76,22 +79,22 @@ func SaveImage(image string, client client.APIClient) (v1.Image, string, error)
 	if err != nil {
 		img, err := daemon.Image(ImageId{name: image}, daemon.WithClient(client))
 		if err != nil {
-			return nil, "", errors.Wrapf(err, "failed to pull image: %s", image)
+			return nil, "", nil, errors.Wrapf(err, "failed to pull image: %s", image)
 		} else {
 			im, _, err := client.ImageInspectWithRaw(context.Background(), image)
 			if err != nil {
-				return nil, "", errors.Wrapf(err, "failed to get local image: %s", image)
+				return nil, "", nil, errors.Wrapf(err, "failed to get local image: %s", image)
 			}
-			path, err = saveOci(im.ID, img, ref, path)
+			path, cleanup, err := saveTar(im.ID, img, ref, path)
 			if err != nil {
-				return nil, "", errors.Wrapf(err, "failed to save image: %s", image)
+				return nil, "", nil, errors.Wrapf(err, "failed to save image: %s", image)
 			}
+			return img, path, cleanup, nil
 		}
-		return img, path, nil
 	} else {
 		img, err := desc.Image()
 		if err != nil {
-			return nil, "", errors.Wrapf(err, "failed to pull image: %s", image)
+			return nil, "", nil, errors.Wrapf(err, "failed to pull image: %s", image)
 		}
 		var digest string
 		identifier := ref.Identifier()
@@ -101,38 +104,70 @@ func SaveImage(image string, client client.APIClient) (v1.Image, string, error)
 			digestHash, _ := img.Digest()
 			digest = digestHash.String()
 		}
-		path, err = saveOci(digest, img, ref, path)
+		path, cleanup, err := saveTar(digest, img, ref, path)
 		if err != nil {
-			return nil, "", errors.Wrapf(err, "failed to save image: %s", image)
+			return nil, "", nil, errors.Wrapf(err, "failed to save image: %s", image)
 		}
-		return img, path, nil
+		return img, path, cleanup, nil
 	}
 }
 
-// saveOci writes the v1.Image img as an OCI Image Layout at path. If a layout
-// already exists at that path, it will add the image to the index.
-func saveOci(digest string, img v1.Image, ref name.Reference, path string) (string, error) {
+func saveTar(digest string, img v1.Image, ref name.Reference, path string) (string, Cleanup, error) {
 	finalPath := strings.Replace(filepath.Join(path, digest), ":", string(os.PathSeparator), 1)
+	tarPath := filepath.Join(finalPath, "archive.tar")
 	skill.Log.Debugf("Copying image to %s", finalPath)
 
-	if _, err := os.Stat(finalPath); !os.IsNotExist(err) {
-		return finalPath, nil
+	if _, err := os.Stat(tarPath); !os.IsNotExist(err) {
+		return finalPath, nil, nil
 	}
 	err := os.MkdirAll(finalPath, os.ModePerm)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
-	p, err := layout.FromPath(finalPath)
-	if err != nil {
-		p, err = layout.Write(finalPath, empty.Index)
-		if err != nil {
-			return "", err
+
+	c := make(chan v1.Update, 200)
+	errchan := make(chan error)
+	go func() {
+		if err := tarball.WriteToFile(tarPath, ref, img, tarball.WithProgress(c)); err != nil {
+			errchan <- errors.Wrapf(err, "failed to write image to tar")
+		}
+		errchan <- nil
+	}()
+
+	cleanup := func() {
+		e := os.Remove(tarPath)
+		if e != nil {
+			skill.Log.Warnf("Failed to delete tmp image archive %s", tarPath)
 		}
 	}
-	if err = p.AppendImage(img); err != nil {
-		return "", err
+
+	var update v1.Update
+	var pp int64
+	for {
+		select {
+		case update = <-c:
+			p := 100 * update.Complete / update.Total
+			if p%10 == 0 && pp != p {
+				skill.Log.WithFields(logrus.Fields{
+					"event":    "copy",
+					"total":    update.Total,
+					"complete": update.Complete,
+				}).Debugf("Copying image %3d%% %s/%s", p, humanize.Bytes(uint64(update.Complete)), humanize.Bytes(uint64(update.Total)))
+				pp = p
+			}
+		case err = <-errchan:
+			if err != nil {
+				return "", cleanup, err
+			} else {
+				skill.Log.WithFields(logrus.Fields{
+					"event":    "copy",
+					"total":    update.Total,
+					"complete": update.Complete,
+				}).Debugf("Copying image completed")
+				return finalPath, cleanup, nil
+			}
+		}
 	}
-	return finalPath, nil
 }
 
 func withAuth() remote.Option {

sbom/detect/detect.go

@@ -17,6 +17,13 @@
 package detect
 
 import (
+	"bufio"
+	"fmt"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+
 	"github.com/anchore/syft/syft/source"
 	"github.com/docker/index-cli-plugin/types"
 )
@@ -26,7 +33,7 @@ type PackageDetector = func(packages []types.Package, image source.Source, lm ty
 var detectors []PackageDetector
 
 func init() {
-	detectors = []PackageDetector{nodePackageDetector}
+	detectors = []PackageDetector{nodePackageDetector()}
 }
 
 func AdditionalPackages(packages []types.Package, image source.Source, lm types.LayerMapping) []types.Package {
@@ -36,3 +43,112 @@ func AdditionalPackages(packages []types.Package, image source.Source, lm types.
 	}
 	return additionalPackages
 }
+
+func stringsNodeDetector(executable string, versionEnvVar string, expr *regexp.Regexp, pkg types.Package) PackageDetector {
+	return func(packages []types.Package, image source.Source, lm types.LayerMapping) []types.Package {
+		// Already found via package manager
+		for _, p := range packages {
+			if purl, err := types.ToPackageUrl(p.Purl); err == nil && purl.Name == pkg.Name {
+				return []types.Package{}
+			}
+		}
+
+		var path []string
+		var version string
+
+		env := image.Image.Metadata.Config.Config.Env
+		for _, e := range env {
+			k := strings.Split(e, "=")[0]
+			v := strings.Split(e, "=")[1]
+			switch k {
+			case versionEnvVar:
+				version = v
+			case "PATH":
+				path = strings.Split(v, ":")
+			}
+		}
+
+		if len(path) > 0 {
+			res, _ := image.FileResolver(source.SquashedScope)
+			for _, p := range path {
+				fp := fmt.Sprintf("%s/%s", p, executable)
+				if locations, err := res.FilesByPath(fp); err == nil && len(locations) > 0 {
+					loc := locations[0]
+
+					if version == "" {
+						f, _ := res.FileContentsByLocation(loc)
+						values := readStrings(f, expr)
+						if len(values) > 0 {
+							version = values[0][1]
+						}
+					}
+
+					if version == "" {
+						continue
+					}
+
+					pkg.Version = version
+					pkg.Purl = types.PackageToPackageUrl(pkg).String()
+					pkg.Locations = []types.Location{{
+						Path:   fp,
+						DiffId: loc.FileSystemID,
+						Digest: lm.ByDiffId[loc.FileSystemID],
+					}}
+					return []types.Package{pkg}
+				}
+			}
+		}
+
+		return []types.Package{}
+	}
+}
+
+var (
+	min   = 6
+	max   = 256
+	ascii = true
+)
+
+func readStrings(reader io.ReadCloser, expr *regexp.Regexp) [][]string {
+	defer reader.Close()
+	in := bufio.NewReader(reader)
+	str := make([]rune, 0, max)
+	filePos := int64(0)
+	verify := func() [][]string {
+		if len(str) >= min {
+			s := string(str)
+			if m := expr.FindAllStringSubmatch(s, -1); len(m) > 0 {
+				return m
+			}
+		}
+		str = str[0:0]
+		return [][]string{}
+	}
+	for {
+		var (
+			r   rune
+			wid int
+			err error
+		)
+		for ; ; filePos += int64(wid) {
+			r, wid, err = in.ReadRune()
+			if err != nil {
+				return [][]string{}
+			}
+			if !strconv.IsPrint(r) || ascii && r >= 0xFF {
+				if d := verify(); len(d) > 0 {
+					return d
+				}
+				continue
+			}
+			// It's printable. Keep it.
+			if len(str) >= max {
+				if d := verify(); len(d) > 0 {
+					return d
+				}
+			}
+			str = append(str, r)
+		}
+	}
+	return [][]string{}
+}

sbom/detect/detect_test.go

@@ -38,7 +38,7 @@ func TestNodeDetector(t *testing.T) {
 		Location:    ociPath,
 	}
 	src, _, _ := source.New(i, nil, nil)
-	packages := nodePackageDetector([]types.Package{}, *src, lm)
+	packages := nodePackageDetector()([]types.Package{}, *src, lm)
 	if len(packages) != 1 {
 		t.Errorf("Expected package missing")
 	}