Implement the extract of hack scripts

Signed-off-by: Chris Suszyński <[email protected]>

cmd/script/main.go

@@ -18,7 +18,7 @@ package main
 
 import (
 	"github.com/wavesoftware/go-commandline"
-	"knative.dev/hack/script/cli"
+	"knative.dev/hack/pkg/inflator/cli"
 )
 
 func main() {

cmd/script/main_test.go

@@ -23,7 +23,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/wavesoftware/go-commandline"
 	main "knative.dev/hack/cmd/script"
-	"knative.dev/hack/script/cli"
+	"knative.dev/hack/pkg/inflator/cli"
 )
 
 func TestMainFn(t *testing.T) {

go.mod

@@ -3,6 +3,7 @@ module knative.dev/hack
 go 1.18
 
 require (
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.5.0
 	github.com/stretchr/testify v1.8.0
 	github.com/wavesoftware/go-commandline v1.0.0

go.sum

@@ -7,6 +7,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

script/cli/app.go → pkg/inflator/cli/app.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/wavesoftware/go-commandline"
-	"knative.dev/hack/script/extract"
+	"knative.dev/hack/pkg/inflator/extract"
 )
 
 // Options to override the commandline for testing purposes.
@@ -22,7 +22,7 @@ func (a App) Command() *cobra.Command {
 			"and provide a source file path to requested script",
 		Example: `
 # In Bash script
-source "$(go run knative.dev/hack/cmd/script library.sh)"`,
+source "$(go run knative.dev/hack/cmd/script@latest library.sh)"`,
 		SilenceUsage: true,
 		Args:         cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, argv []string) error {

pkg/inflator/cli/app_test.go

@@ -0,0 +1,29 @@
+package cli_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"knative.dev/hack/pkg/inflator/cli"
+	"knative.dev/hack/pkg/inflator/extract"
+)
+
+func TestApp(t *testing.T) {
+	tmpdir := t.TempDir()
+	t.Setenv(extract.ArtifactsEnvVar, tmpdir)
+	c := cli.App{}.Command()
+	var (
+		outb bytes.Buffer
+		errb bytes.Buffer
+	)
+	c.SetOut(&outb)
+	c.SetErr(&errb)
+	c.SetArgs([]string{"e2e-tests.sh"})
+	err := c.Execute()
+
+	require.NoError(t, err)
+	assert.Equal(t, outb.String(), tmpdir+"/hack-scripts/e2e-tests.sh\n")
+	assert.Equal(t, errb.String(), "")
+}

script/extract/errors.go → pkg/inflator/extract/errors.go

@@ -1,8 +1,9 @@
 package extract
 
 import (
-	"errors"
 	"fmt"
+
+	"github.com/pkg/errors"
 )
 
 var (
@@ -14,8 +15,11 @@ var (
 )
 
 func wrapErr(err error, target error) error {
+	if err == nil {
+		return nil
+	}
 	if errors.Is(err, target) {
 		return err
 	}
-	return fmt.Errorf("%w: %v", target, err)
+	return errors.WithStack(fmt.Errorf("%w: %v", target, err))
 }

pkg/inflator/extract/extract.go

@@ -0,0 +1,135 @@
+package extract
+
+import (
+	"io/fs"
+	"math"
+	"os"
+	"path"
+	"strings"
+
+	"knative.dev/hack"
+)
+
+const (
+	// ArtifactsEnvVar is the name of the environment variable that points
+	// to ARTIFACTS directory.
+	ArtifactsEnvVar = "ARTIFACTS"
+	// PermOwnerWrite is the permission bits for owner write.
+	PermOwnerWrite = 0o200
+	// PermAllExecutable is the permission bits for executable.
+	PermAllExecutable = 0o111
+)
+
+// Printer is an interface for printing messages.
+type Printer interface {
+	Print(i ...interface{})
+	Println(i ...interface{})
+	Printf(format string, i ...interface{})
+	PrintErr(i ...interface{})
+	PrintErrln(i ...interface{})
+	PrintErrf(format string, i ...interface{})
+}
+
+// Operation is the main extract object that can extract scripts.
+type Operation struct {
+	// ScriptName is the name of the script to extract.
+	ScriptName string
+	// Verbose will print more information.
+	Verbose bool
+}
+
+// Extract will extract a script from the library to a temporary directory and
+// provide the file path to it.
+func (o Operation) Extract(prtr Printer) error {
+	l := logger{o.Verbose, prtr}
+	artifactsDir := os.Getenv(ArtifactsEnvVar)
+	if artifactsDir == "" {
+		var err error
+		if artifactsDir, err = os.MkdirTemp("", "knative.*"); err != nil {
+			return wrapErr(err, ErrBug)
+		}
+	}
+	hackRootDir := path.Join(artifactsDir, "hack-scripts")
+	l.debugf("Extracting hack scripts to directory: %s", hackRootDir)
+	if err := copyDir(l, hack.Scripts, hackRootDir, "."); err != nil {
+		return err
+	}
+	scriptPath := path.Join(hackRootDir, o.ScriptName)
+	l.Println(scriptPath)
+	return nil
+}
+
+func copyDir(l logger, inputFS fs.ReadDirFS, destRootDir, dir string) error {
+	return wrapErr(fs.WalkDir(inputFS, dir, func(filePath string, dirEntry fs.DirEntry, err error) error {
+		if err != nil {
+			return wrapErr(err, ErrBug)
+		}
+		return copyFile(l, inputFS, destRootDir, filePath, dirEntry)
+	}), ErrUnexpected)
+}
+
+func copyFile(
+	l logger,
+	inputFS fs.ReadDirFS,
+	destRootDir, filePath string,
+	dirEntry fs.DirEntry,
+) error {
+	inputFI, err := dirEntry.Info()
+	if err != nil {
+		return wrapErr(err, ErrBug)
+	}
+
+	destPath := path.Join(destRootDir, filePath)
+	perm := inputFI.Mode().Perm()
+
+	perm |= PermOwnerWrite
+	if dirEntry.IsDir() {
+		perm |= PermAllExecutable
+		if err = os.MkdirAll(destPath, perm); err != nil {
+			return wrapErr(err, ErrUnexpected)
+		}
+		return nil
+	}
+
+	var (
+		inputCS checksum
+		destCS  checksum
+		destFI  fs.FileInfo
+		bytes   []byte
+	)
+	inputCS = asChecksum(inputFI)
+	if destFI, err = os.Stat(destPath); err != nil {
+		if !os.IsNotExist(err) {
+			return wrapErr(err, ErrUnexpected)
+		}
+	} else {
+		destCS = asChecksum(destFI)
+	}
+	if inputCS == destCS {
+		l.debugf("%-30s up-to-date", filePath)
+		return nil
+	}
+	if bytes, err = fs.ReadFile(inputFS, filePath); err != nil {
+		return wrapErr(err, ErrBug)
+	}
+	if err = os.WriteFile(destPath, bytes, perm); err != nil {
+		return wrapErr(err, ErrUnexpected)
+	}
+
+	sizeKB := int(inputFI.Size() / 1024)
+	size5k := int(math.Ceil(float64(sizeKB) / 5))
+	l.debugf("%-30s %3d KiB %s", filePath, sizeKB, strings.Repeat("+", size5k))
+	return nil
+}
+
+func asChecksum(f fs.FileInfo) checksum {
+	return checksum{
+		exists: true,
+		size:   f.Size(),
+	}
+}
+
+type checksum struct {
+	exists bool
+	size   int64
+}

pkg/inflator/extract/extract_test.go

@@ -0,0 +1,84 @@
+package extract_test
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"knative.dev/hack/pkg/inflator/extract"
+)
+
+func TestExtract(t *testing.T) {
+	tmpdir := t.TempDir()
+	t.Setenv(extract.ArtifactsEnvVar, tmpdir)
+	op := extract.Operation{
+		ScriptName: "library.sh",
+		Verbose:    true,
+	}
+	prtr := &testPrinter{}
+	err := op.Extract(prtr)
+	require.NoError(t, err)
+	assert.Equal(t, prtr.out.String(), tmpdir+"/hack-scripts/library.sh\n")
+	assert.Equal(t,
+		`[hack] Extracting hack scripts to directory: /tmp/x/hack-scripts
+[hack] codegen-library.sh               1 KiB +
+[hack] e2e-tests.sh                     6 KiB ++
+[hack] infra-library.sh                 5 KiB +
+[hack] library.sh                      33 KiB +++++++
+[hack] microbenchmarks.sh               2 KiB +
+[hack] performance-tests.sh             6 KiB ++
+[hack] presubmit-tests.sh              12 KiB +++
+[hack] release.sh                      26 KiB ++++++
+[hack] shellcheck-presubmit.sh          1 KiB +
+`, strings.ReplaceAll(prtr.err.String(), tmpdir, "/tmp/x"))
+
+	// second time should be a no-op
+	prtr = &testPrinter{}
+	err = op.Extract(prtr)
+	require.NoError(t, err)
+	assert.Equal(t, prtr.out.String(), tmpdir+"/hack-scripts/library.sh\n")
+	assert.Equal(t,
+		`[hack] Extracting hack scripts to directory: /tmp/x/hack-scripts
+[hack] codegen-library.sh             up-to-date
+[hack] e2e-tests.sh                   up-to-date
+[hack] infra-library.sh               up-to-date
+[hack] library.sh                     up-to-date
+[hack] microbenchmarks.sh             up-to-date
+[hack] performance-tests.sh           up-to-date
+[hack] presubmit-tests.sh             up-to-date
+[hack] release.sh                     up-to-date
+[hack] shellcheck-presubmit.sh        up-to-date
+`, strings.ReplaceAll(prtr.err.String(), tmpdir, "/tmp/x"))
+}
+
+type testPrinter struct {
+	out bytes.Buffer
+	err bytes.Buffer
+}
+
+func (t *testPrinter) Print(i ...interface{}) {
+	_, _ = fmt.Fprint(&t.out, i...)
+}
+
+func (t *testPrinter) Println(i ...interface{}) {
+	t.Print(fmt.Sprintln(i...))
+}
+
+func (t *testPrinter) Printf(format string, i ...interface{}) {
+	t.Print(fmt.Sprintf(format, i...))
+}
+
+func (t *testPrinter) PrintErr(i ...interface{}) {
+	_, _ = fmt.Fprint(&t.err, i...)
+}
+
+func (t *testPrinter) PrintErrln(i ...interface{}) {
+	t.PrintErr(fmt.Sprintln(i...))
+}
+
+func (t *testPrinter) PrintErrf(format string, i ...interface{}) {
+	t.PrintErr(fmt.Sprintf(format, i...))
+}