🎁 Source embedded hack scripts (#222)

* Source embedded hack scripts

* Implement the extract of hack scripts

Signed-off-by: Chris Suszyński <[email protected]>

* Remove k8s.io/apimachinery dep

* Change the extract dir to be constant

* TestExtract should not check exact sizes of extracted files

* Ensure the manual verbose is set for all unit tests

* Default E2E_CLUSTER_REGION us-central1 to simplify unit testing

---------

Signed-off-by: Chris Suszyński <[email protected]>

cmd/script/main.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2022 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/wavesoftware/go-commandline"
+	"knative.dev/hack/pkg/inflator/cli"
+)
+
+func main() {
+	commandline.New(new(cli.App)).ExecuteOrDie(cli.Options...)
+}
+
+// RunMain is used by tests to run the main function.
+func RunMain() { // nolint:deadcode
+	main()
+}

cmd/script/main_test.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2022 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/wavesoftware/go-commandline"
+	main "knative.dev/hack/cmd/script"
+	"knative.dev/hack/pkg/inflator/cli"
+)
+
+func TestMainFn(t *testing.T) {
+	var buf bytes.Buffer
+	var retcode *int
+	withOptions(
+		func() {
+			main.RunMain()
+		},
+		commandline.WithArgs("--help"),
+		commandline.WithOutput(&buf),
+		commandline.WithExit(func(c int) {
+			retcode = &c
+		}),
+	)
+	assert.Nil(t, retcode)
+	assert.Contains(t, buf.String(), "Script will extract Hack scripts")
+}
+
+func withOptions(fn func(), options ...commandline.Option) {
+	prev := cli.Options
+	cli.Options = options
+	defer func(p []commandline.Option) {
+		cli.Options = p
+	}(prev)
+	fn()
+}

hack.go → embed.go

@@ -1,8 +1,5 @@
-//go:build hack
-// +build hack
-
 /*
-Copyright 2020 The Knative Authors
+Copyright 2022 The Knative Authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,7 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// package hack is a collection of scripts used to bootstrap CI processes and
-// other vital entrypoint functionality.
-
 package hack
+
+import "embed"
+
+//go:embed *.sh
+var Scripts embed.FS

embed_test.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2022 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hack_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"knative.dev/hack"
+)
+
+func TestScriptsAreEmbedded(t *testing.T) {
+	entries, err := hack.Scripts.ReadDir(".")
+	require.NoError(t, err)
+	names := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		names = append(names, entry.Name())
+		if !entry.IsDir() {
+			fi, err := entry.Info()
+			require.NoError(t, err)
+			assert.Greater(t, fi.Size(), int64(0))
+		}
+	}
+	requiredLibs := []string{
+		"codegen-library.sh",
+		"e2e-tests.sh",
+		"infra-library.sh",
+		"library.sh",
+		"microbenchmarks.sh",
+		"performance-tests.sh",
+		"presubmit-tests.sh",
+		"release.sh",
+		"shellcheck-presubmit.sh",
+	}
+	for _, lib := range requiredLibs {
+		assert.Contains(t, names, lib)
+	}
+}

go.mod

@@ -1,3 +1,23 @@
 module knative.dev/hack
 
 go 1.18
+
+require (
+	github.com/pkg/errors v0.9.1
+	github.com/spf13/cobra v1.5.0
+	github.com/stretchr/testify v1.8.0
+	github.com/wavesoftware/go-commandline v1.0.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/wavesoftware/go-retcode v1.0.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,42 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/wavesoftware/go-commandline v1.0.0 h1:n7nrFr1unfiUcF7shA1rYf+YhXB12pY8uNYqPgFsHio=
+github.com/wavesoftware/go-commandline v1.0.0/go.mod h1:C9yRtwZxJSck99kk6SRRkOtC2ppQF/KDRy0yrzWJuHU=
+github.com/wavesoftware/go-retcode v1.0.0 h1:Z53+VpIHMvRMtjS6jPScdihbAN1ks3lIJ5Mj32gCpno=
+github.com/wavesoftware/go-retcode v1.0.0/go.mod h1:BLqIIXhB/PQ+izkkRGfSQgu95BDtMmUBuvTJ/gkSWVM=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=

go.work.sum

@@ -1,12 +1,2 @@
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
-k8s.io/apimachinery v0.20.6 h1:R5p3SlhaABYShQSO6LpPsYHjV05Q+79eBUR0Ut/f4tk=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
-sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06 h1:zD2IemQ4LmOcAumeiyDWXKUI2SO0NYDe3H6QGvPOVgU=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.3 h1:4oyYo8NREp49LBBhKxEqCulFjg26rawYKrnCmg+Sr6c=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

infra-library.sh

@@ -157,7 +157,7 @@ function create_gke_test_cluster() {
     --v=1 \
     --network=e2e-network \
     --boskos-acquire-timeout-seconds=1200 \
-    --region="${E2E_CLUSTER_REGION},us-east1,us-west1" \
+    --region="${E2E_CLUSTER_REGION:-us-central1},us-east1,us-west1" \
     --gcloud-extra-flags="${_extra_gcloud_flags[*]}" \
     --retryable-error-patterns='.*does not have enough resources available to fulfill.*,.*only \\d+ nodes out of \\d+ have registered; this is likely due to Nodes failing to start correctly.*,.*All cluster resources were brought up.+ but: component .+ from endpoint .+ is unhealthy.*' \
     --test=exec \

pkg/inflator/cli/app.go

@@ -0,0 +1,42 @@
+package cli
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/wavesoftware/go-commandline"
+	"knative.dev/hack/pkg/inflator/extract"
+)
+
+// Options to override the commandline for testing purposes.
+var Options []commandline.Option //nolint:gochecknoglobals
+
+type App struct{}
+
+func (a App) Command() *cobra.Command {
+	fl := &flags{}
+	c := &cobra.Command{
+		Use:   "script library.sh",
+		Short: "Script is a tool for running Hack scripts",
+		Long: "Script will extract Hack scripts to a temporary directory, " +
+			"and provide a source file path to requested script",
+		Example: `
+# In Bash script
+source "$(go run knative.dev/hack/cmd/script@latest library.sh)"`,
+		SilenceUsage: true,
+		Args:         cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, argv []string) error {
+			op := createOperation(fl, argv)
+			return op.Extract(cmd)
+		},
+	}
+	c.SetOut(os.Stdout)
+	return fl.withFlags(c)
+}
+
+func createOperation(fl *flags, argv []string) extract.Operation {
+	return extract.Operation{
+		ScriptName: argv[0],
+		Verbose:    fl.verbose,
+	}
+}

pkg/inflator/cli/app_test.go

@@ -0,0 +1,30 @@
+package cli_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"knative.dev/hack/pkg/inflator/cli"
+	"knative.dev/hack/pkg/inflator/extract"
+)
+
+func TestApp(t *testing.T) {
+	tmpdir := t.TempDir()
+	t.Setenv(extract.HackScriptsDirEnvVar, tmpdir)
+	t.Setenv(cli.ManualVerboseEnvVar, "true")
+	c := cli.App{}.Command()
+	var (
+		outb bytes.Buffer
+		errb bytes.Buffer
+	)
+	c.SetOut(&outb)
+	c.SetErr(&errb)
+	c.SetArgs([]string{"e2e-tests.sh"})
+	err := c.Execute()
+
+	require.NoError(t, err)
+	assert.Equal(t, outb.String(), tmpdir+"/e2e-tests.sh\n")
+	assert.Equal(t, errb.String(), "")
+}

pkg/inflator/cli/flags.go

@@ -0,0 +1,33 @@
+package cli
+
+import (
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+const (
+	// ManualVerboseEnvVar is the environment variable that can be set to disable
+	// automatic verbose mode on CI servers.
+	ManualVerboseEnvVar = "KNATIVE_HACK_SCRIPT_MANUAL_VERBOSE"
+)
+
+type flags struct {
+	verbose bool
+}
+
+func (f *flags) withFlags(c *cobra.Command) *cobra.Command {
+	fl := c.PersistentFlags()
+	fl.BoolVarP(&f.verbose, "verbose", "v", isCiServer(), "Print verbose output on Stderr")
+	return c
+}
+
+func isCiServer() bool {
+	if strings.HasPrefix(strings.ToLower(os.Getenv(ManualVerboseEnvVar)), "t") {
+		return false
+	}
+	return os.Getenv("CI") != "" ||
+		os.Getenv("BUILD_ID") != "" ||
+		os.Getenv("PROW_JOB_ID") != ""
+}

pkg/inflator/extract/errors.go

@@ -0,0 +1,25 @@
+package extract
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	// ErrBug is an error that indicates a bug in the code.
+	ErrBug = errors.New("probably a bug in the code")
+
+	// ErrUnexpected is an error that indicates an unexpected situation.
+	ErrUnexpected = errors.New("unexpected situation")
+)
+
+func wrapErr(err error, target error) error {
+	if err == nil {
+		return nil
+	}
+	if errors.Is(err, target) {
+		return err
+	}
+	return errors.WithStack(fmt.Errorf("%w: %v", target, err))
+}