Merge remote-tracking branch 'upstream/main' into feature/source-embe…

…dded-hacks

Conflicts fixed:

 * test/unit/sharedlib_test.go

OWNERS_ALIASES

@@ -53,6 +53,18 @@ aliases:
   - matzew
   - odacremolbap
   - pierDipi
+  func-reviewers:
+  - jrangelramos
+  - nainaz
+  func-writers:
+  - lance
+  - lkingland
+  - matejvasek
+  - salaboy
+  - zroubalik
+  functions-wg-leads:
+  - lance
+  - salaboy
   knative-admin:
   - csantanapr
   - dprotaso
@@ -67,11 +79,11 @@ aliases:
   - kvmware
   - lance
   - lionelvillard
+  - mchmarny
   - nak3
   - pmorie
   - psschwei
   - smoser-ibm
-  - spencerdillard
   - thisisnotapril
   - upodroid
   - vaikas
@@ -145,7 +157,6 @@ aliases:
   productivity-reviewers:
   - albertomilan
   - evankanderson
-  - gerardo-lc
   - mgencur
   - shinigambit
   productivity-wg-leads:
@@ -201,8 +212,8 @@ aliases:
   - zroubalik
   trademark-committee:
   - evankanderson
+  - mchmarny
   - smoser-ibm
-  - spencerdillard
   ux-wg-leads:
   - abrennan89
   - snneji

presubmit-tests.sh

@@ -139,14 +139,7 @@ function __build_test_runner_for_module() {
   # Don't merge these two lines, or return code will always be 0.
   # Get all build tags in go code (ignore /vendor, /hack and /third_party)
   local tags
-  tags="$(grep -I -r '// +build' . | grep -v '/vendor/' | \
-    grep -v '/hack/' | \
-    grep -v '/third_party' | \
-    cut -f3 -d' ' | \
-    tr ',' '\n' | \
-    sort | uniq | \
-    grep -v '^!' | \
-    paste -s -d, /dev/stdin)"
+  tags="$(go run knative.dev/test-infra/tools/go-ls-tags@latest --joiner=,)"
   local go_pkg_dirs
   go_pkg_dirs="$(go list -tags "${tags}" ./...)" || return $?
   if [[ -z "${go_pkg_dirs}" ]]; then

release.sh

@@ -314,7 +314,7 @@ function build_from_source() {
 
 # Build a release from source.
 function sign_release() {
-  if [ -z "${SIGN_IMAGES:-}" ]; then # Temporary Feature Gate
+  if (( ! IS_PROW )); then # This function can't be run by devs on their laptops
     return 0
   fi
 
@@ -331,26 +331,29 @@ function sign_release() {
     zip files.zip ${FILES}
     rcodesign notary-submit files.zip --api-key-path="${APPLE_NOTARY_API_KEY}" --wait
     sha256sum ${ARTIFACTS_TO_PUBLISH//checksums.txt/} > checksums.txt
+    echo "🧮     Post Notarization Checksum:"
+    cat checksums.txt
   fi
 
+  ID_TOKEN=$(gcloud auth print-identity-token --audiences=sigstore \
+    --include-email \
+    --impersonate-service-account="${SIGNING_IDENTITY}")
+  echo "Signing Images with the identity ${SIGNING_IDENTITY}"
   ## Sign the images with cosign
-  ## For now, check if ko has created imagerefs.txt file. In the future, missing image refs will break
-  ## the release for all jobs that publish images.
   if [[ -f "imagerefs.txt" ]]; then
-      echo "Signing Images with the identity ${SIGNING_IDENTITY}"
-      COSIGN_EXPERIMENTAL=1 cosign sign $(cat imagerefs.txt) --recursive --identity-token="$(
-        gcloud auth print-identity-token --audiences=sigstore \
-        --include-email \
-        --impersonate-service-account="${SIGNING_IDENTITY}")"
+      COSIGN_EXPERIMENTAL=1 cosign sign $(cat imagerefs.txt) --recursive --identity-token="${ID_TOKEN}"
+      if  [ -n "${ATTEST_IMAGES:-}" ]; then # Temporary Feature Gate
+        provenance-generator --clone-log=/logs/clone.json \
+          --image-refs=imagerefs.txt --output=attestation.json
+        COSIGN_EXPERIMENTAL=1 cosign attest $(cat imagerefs.txt) --recursive --identity-token="${ID_TOKEN}" \
+          --predicate=attestation.json --type=slsaprovenance
+      fi
   fi
 
   ## Check if there is checksums.txt file. If so, sign the checksum file
   if [[ -f "checksums.txt" ]]; then
       echo "Signing Images with the identity ${SIGNING_IDENTITY}"
-      COSIGN_EXPERIMENTAL=1 cosign sign-blob checksums.txt --output-signature=checksums.txt.sig --output-certificate=checksums.txt.pem --identity-token="$(
-        gcloud auth print-identity-token --audiences=sigstore \
-        --include-email \
-        --impersonate-service-account="${SIGNING_IDENTITY}")"
+      COSIGN_EXPERIMENTAL=1 cosign sign-blob checksums.txt --output-signature=checksums.txt.sig --output-certificate=checksums.txt.pem --identity-token="${ID_TOKEN}"
       ARTIFACTS_TO_PUBLISH="${ARTIFACTS_TO_PUBLISH} checksums.txt.sig checksums.txt.pem"
   fi
 }

test/unit/sharedlib_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"embed"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path"
@@ -127,7 +128,15 @@ func (tc testCase) validRetcode(t TestingT, gotRetcode int) {
 	}
 }
 
-type scriptlet func(t TestingT) string
+type scriptlet interface {
+	scriptlet(t TestingT) string
+}
+
+type fnScriptlet func(t TestingT) string
+
+func (f fnScriptlet) scriptlet(t TestingT) string {
+	return f(t)
+}
 
 func newShellScript(scriptlets ...scriptlet) shellScript {
 	return shellScript{
@@ -142,28 +151,28 @@ type shellScript struct {
 }
 
 func loadFile(names ...string) scriptlet {
-	return func(t TestingT) string {
+	return fnScriptlet(func(t TestingT) string {
 		sc := make([]scriptlet, 0, len(names))
 		for i := range names {
 			name := names[i]
-			sc = append(sc, func(t TestingT) string {
+			sc = append(sc, fnScriptlet(func(t TestingT) string {
 				byts, err := scripts.ReadFile(path.Join("scripts", name))
 				require.NoError(t, err)
 				return string(byts)
-			})
+			}))
 		}
 		src := make([]string, len(sc))
 		for i, s := range sc {
-			src[i] = s(t)
+			src[i] = s.scriptlet(t)
 		}
 		return strings.Join(src, "\n")
-	}
+	})
 }
 
 func instructions(inst ...string) scriptlet {
-	return func(t TestingT) string {
+	return fnScriptlet(func(t TestingT) string {
 		return strings.Join(inst, "\n")
-	}
+	})
 }
 
 type simply string
@@ -190,7 +199,7 @@ func (o callOriginal) Invocations(bin string) []string {
 }
 
 func mockBinary(name string, responses ...response) scriptlet {
-	return func(t TestingT) string {
+	return fnScriptlet(func(t TestingT) string {
 		code := make([]string, 0, len(responses)*10)
 		code = append(code,
 			fmt.Sprintf(`cat > "${TMPPATH}/%s" <<'EOF'`, name),
@@ -208,7 +217,7 @@ func mockBinary(name string, responses ...response) scriptlet {
 			fmt.Sprintf(`chmod +x "${TMPPATH}/%s"`, name),
 		)
 		return strings.Join(code, "\n") + "\n"
-	}
+	})
 }
 
 type invocations interface {
@@ -239,8 +248,11 @@ func (a anyArgs) String() string {
 }
 
 func mockGo(responses ...response) scriptlet {
+	lstags := "knative.dev/test-infra/tools/go-ls-tags@latest"
+	modscope := "knative.dev/test-infra/tools/modscope@latest"
 	callOriginals := []args{
-		startsWith{"run knative.dev/test-infra/tools/modscope@latest"},
+		startsWith{"run " + lstags},
+		startsWith{"run " + modscope},
 		startsWith{"list"},
 		startsWith{"env"},
 		startsWith{"version"},
@@ -249,7 +261,13 @@ func mockGo(responses ...response) scriptlet {
 	for i, co := range callOriginals {
 		originalResponses[i] = response{co, callOriginal{}}
 	}
-	return mockBinary("go", append(originalResponses, responses...)...)
+	return prefetchScriptlet{
+		delegate: mockBinary("go", append(originalResponses, responses...)...),
+		prefetchers: []prefetcher{
+			goRunHelpPrefetcher(lstags),
+			goRunHelpPrefetcher(modscope),
+		},
+	}
 }
 
 func mockKubectl(responses ...response) scriptlet {
@@ -272,13 +290,13 @@ func fakeProwJob() scriptlet {
 }
 
 func union(scriptlets ...scriptlet) scriptlet {
-	return func(t TestingT) string {
+	return fnScriptlet(func(t TestingT) string {
 		code := make([]string, 0, len(scriptlets)*10)
 		for _, s := range scriptlets {
-			code = append(code, s(t))
+			code = append(code, s.scriptlet(t))
 		}
 		return strings.Join(code, "\n")
-	}
+	})
 }
 
 type TestingT interface {
@@ -288,6 +306,7 @@ type TestingT interface {
 }
 
 func (s shellScript) run(t TestingT, commands []string) (int, string, string, string) {
+	s.prefetch(t)
 	src := s.source(t, commands)
 	sf := s.write(t, src)
 	defer func(name string) {
@@ -316,7 +335,7 @@ export PATH="${TMPPATH}:${PATH}"
 `, t.TempDir())
 	bashShebang := "#!/usr/bin/env bash\n"
 	for _, sclet := range s.scriptlets {
-		source += "\n" + strings.TrimPrefix(sclet(t), bashShebang) + "\n"
+		source += "\n" + strings.TrimPrefix(sclet.scriptlet(t), bashShebang) + "\n"
 	}
 	source = bashShebang + "\n" +
 		bashQuotesRx.ReplaceAllStringFunc(source, func(in string) string {
@@ -341,6 +360,66 @@ func (s shellScript) write(t TestingT, src string) string {
 	return p
 }
 
+type prefetcher interface {
+	prefetch(t TestingT)
+}
+
+type fnPrefetcher func(t TestingT)
+
+func (f fnPrefetcher) prefetch(t TestingT) {
+	f(t)
+}
+
+// goRunHelpPrefetcher will call `go run tool --help` before the testing starts.
+// This is to ensure the given tool is downloaded and compiled, so the download
+// and compilation messages, which go prints will not influence the test.
+func goRunHelpPrefetcher(tool string) prefetcher {
+	return fnPrefetcher(func(t TestingT) {
+		c := exec.Command("go", "run", tool, "--help")
+		var (
+			stdout, stderr io.ReadCloser
+			err            error
+		)
+		stdout, err = c.StdoutPipe()
+		require.NoError(t, err)
+		stderr, err = c.StderrPipe()
+		require.NoError(t, err)
+		err = c.Run()
+		if err != nil {
+			stdBytes, merr := io.ReadAll(stdout)
+			require.NoError(t, merr)
+			errBytes, rerr := io.ReadAll(stderr)
+			require.NoError(t, rerr)
+			require.NoError(t, err,
+				"------\nSTDOUT\n------", string(stdBytes),
+				"------\nSTDERR\n------", string(errBytes))
+		}
+	})
+}
+
+type prefetchScriptlet struct {
+	delegate    scriptlet
+	prefetchers []prefetcher
+}
+
+func (p prefetchScriptlet) scriptlet(t TestingT) string {
+	return p.delegate.scriptlet(t)
+}
+
+func (p prefetchScriptlet) prefetch(t TestingT) {
+	for _, pr := range p.prefetchers {
+		pr.prefetch(t)
+	}
+}
+
+func (s shellScript) prefetch(t TestingT) {
+	for _, sclet := range s.scriptlets {
+		if pf, ok := sclet.(prefetcher); ok {
+			pf.prefetch(t)
+		}
+	}
+}
+
 func currentDir() string {
 	_, file, _, _ := runtime.Caller(0)
 	return path.Dir(file)