Merge remote-tracking branch 'origin/master' into issue5540

.github/workflows/go.yml

@@ -17,8 +17,8 @@ jobs:
     outputs:
       doc: ${{ steps.filter.outputs.doc }}
     steps:
-    - uses: actions/checkout@v3
-    - uses: dorny/paths-filter@v2
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
       id: filter
       with:
         filters: |

api/filters/patchjson6902/patchjson6902.go

@@ -6,7 +6,7 @@ package patchjson6902
 import (
 	"strings"
 
-	jsonpatch "github.com/evanphx/json-patch"
+	jsonpatch "gopkg.in/evanphx/json-patch.v4"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 	k8syaml "sigs.k8s.io/yaml"

api/go.mod

@@ -3,18 +3,18 @@ module sigs.k8s.io/kustomize/api
 go 1.20
 
 require (
-	github.com/evanphx/json-patch v4.12.0+incompatible
+	github.com/blang/semver/v4 v4.0.0
 	github.com/go-errors/errors v1.4.2
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/stretchr/testify v1.8.1
 	go.uber.org/goleak v1.3.0
+	gopkg.in/evanphx/json-patch.v4 v4.12.0
 	k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961
 	sigs.k8s.io/kustomize/kyaml v0.16.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
-	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect

api/go.sum

@@ -7,8 +7,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
-github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
@@ -77,6 +75,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

api/internal/plugins/builtinconfig/loaddefaultconfig.go

@@ -33,7 +33,7 @@ func loadDefaultConfig(
 // makeTransformerConfigFromBytes returns a TransformerConfig object from bytes
 func makeTransformerConfigFromBytes(data []byte) (*TransformerConfig, error) {
 	var t TransformerConfig
-	err := yaml.Unmarshal(data, &t)
+	err := yaml.UnmarshalStrict(data, &t)
 	if err != nil {
 		return nil, err
 	}

api/internal/plugins/builtinconfig/loaddefaultconfig_test.go

@@ -5,6 +5,7 @@ package builtinconfig
 
 import (
 	"reflect"
+	"strings"
 	"testing"
 
 	"sigs.k8s.io/kustomize/api/internal/loader"
@@ -44,3 +45,55 @@ namePrefix:
 		t.Fatalf("expected %v\n but go6t %v\n", expected, tCfg)
 	}
 }
+
+func TestLoadDefaultConfigsFromFilesWithMissingFields(t *testing.T) {
+	fSys := filesys.MakeFsInMemory()
+	filePathContainsTypo := "config_contains_typo.yaml"
+	if err := fSys.WriteFile(filePathContainsTypo, []byte(`
+namoPrefix:
+- path: nameprefix/path
+  kind: SomeKind
+`)); err != nil {
+		t.Fatal(err)
+	}
+	ldr, err := loader.NewLoader(
+		loader.RestrictionRootOnly, filesys.Separator, fSys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	errMsg := "error unmarshaling JSON: while decoding JSON: json: unknown field"
+	_, err = loadDefaultConfig(ldr, []string{filePathContainsTypo})
+	if err == nil {
+		t.Fatalf("expected to fail unmarshal yaml, but got nil %s", filePathContainsTypo)
+	}
+	if !strings.Contains(err.Error(), errMsg) {
+		t.Fatalf("expected error %s, but got %s", errMsg, err)
+	}
+}
+
+// please remove this failing test after implements the labels support
+func TestLoadDefaultConfigsFromFilesWithMissingFieldsLabels(t *testing.T) {
+	fSys := filesys.MakeFsInMemory()
+	filePathContainsTypo := "config_contains_typo.yaml"
+	if err := fSys.WriteFile(filePathContainsTypo, []byte(`
+labels:
+  - path: spec/podTemplate/metadata/labels
+    create: true
+    kind: FlinkDeployment
+`)); err != nil {
+		t.Fatal(err)
+	}
+	ldr, err := loader.NewLoader(
+		loader.RestrictionRootOnly, filesys.Separator, fSys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	errMsg := "error unmarshaling JSON: while decoding JSON: json: unknown field"
+	_, err = loadDefaultConfig(ldr, []string{filePathContainsTypo})
+	if err == nil {
+		t.Fatalf("expected to fail unmarshal yaml, but got nil %s", filePathContainsTypo)
+	}
+	if !strings.Contains(err.Error(), errMsg) {
+		t.Fatalf("expected error %s, but got %s", errMsg, err)
+	}
+}

api/internal/plugins/execplugin/execplugin.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/google/shlex"
+	"k8s.io/klog"
 
 	"sigs.k8s.io/kustomize/api/internal/plugins/utils"
 	"sigs.k8s.io/kustomize/api/resmap"
@@ -21,6 +22,7 @@ import (
 
 const (
 	tmpConfigFilePrefix = "kust-plugin-config-"
+	maxArgStringLength  = 131071
 )
 
 // ExecPlugin record the name and args of an executable
@@ -169,23 +171,35 @@ func (p *ExecPlugin) invokePlugin(input []byte) ([]byte, error) {
 		p.path, append([]string{f.Name()}, p.args...)...)
 	cmd.Env = p.getEnv()
 	cmd.Stdin = bytes.NewReader(input)
-	cmd.Stderr = os.Stderr
+	var stdErr bytes.Buffer
+	cmd.Stderr = &stdErr
 	if _, err := os.Stat(p.h.Loader().Root()); err == nil {
 		cmd.Dir = p.h.Loader().Root()
 	}
 	result, err := cmd.Output()
 	if err != nil {
 		return nil, errors.WrapPrefixf(
-			err, "failure in plugin configured via %s; %v",
-			f.Name(), err.Error())
+			fmt.Errorf("failure in plugin configured via %s; %w",
+				f.Name(), err), stdErr.String())
 	}
 	return result, os.Remove(f.Name())
 }
 
 func (p *ExecPlugin) getEnv() []string {
 	env := os.Environ()
-	env = append(env,
-		"KUSTOMIZE_PLUGIN_CONFIG_STRING="+string(p.cfg),
-		"KUSTOMIZE_PLUGIN_CONFIG_ROOT="+p.h.Loader().Root())
+	pluginConfigString := "KUSTOMIZE_PLUGIN_CONFIG_STRING=" + string(p.cfg)
+	if len(pluginConfigString) <= maxArgStringLength {
+		env = append(env, pluginConfigString)
+	} else {
+		klog.Warningf("KUSTOMIZE_PLUGIN_CONFIG_STRING exceeds hard limit of %d characters, the environment variable "+
+			"will be omitted", maxArgStringLength)
+	}
+	pluginConfigRoot := "KUSTOMIZE_PLUGIN_CONFIG_ROOT=" + p.h.Loader().Root()
+	if len(pluginConfigRoot) <= maxArgStringLength {
+		env = append(env, pluginConfigRoot)
+	} else {
+		klog.Warningf("KUSTOMIZE_PLUGIN_CONFIG_ROOT exceeds hard limit of %d characters, the environment variable "+
+			"will be omitted", maxArgStringLength)
+	}
 	return env
 }

api/internal/plugins/execplugin/execplugin_test.go

@@ -20,6 +20,12 @@ import (
 	"sigs.k8s.io/kustomize/kyaml/filesys"
 )
 
+const (
+	expectedLargeConfigMap = `{"apiVersion":"v1","data":{"password":"password","username":"user"},"kind":"ConfigMap",` +
+		`"metadata":{"annotations":{"internal.config.kubernetes.io/generatorBehavior":"unspecified",` +
+		`"internal.config.kubernetes.io/needsHashSuffix":"enabled"},"name":"example-configmap-test"}}`
+)
+
 func TestExecPluginConfig(t *testing.T) {
 	fSys := filesys.MakeFsInMemory()
 	err := fSys.WriteFile("sed-input.txt", []byte(`
@@ -125,3 +131,104 @@ func TestExecPlugin_ErrIfNotExecutable(t *testing.T) {
 		t.Fatalf("unexpected err: %v", err)
 	}
 }
+
+// TestExecPluginLarge loads PluginConfigs of various (large) sizes. It tests if the env variable is kept below the
+// maximum of 131072 bytes.
+func TestExecPluginLarge(t *testing.T) {
+	// Skip this test on windows.
+	if runtime.GOOS == "windows" {
+		t.Skipf("always returns nil on Windows")
+	}
+
+	// Add executable plugin.
+	srcRoot, err := utils.DeterminePluginSrcRoot(filesys.MakeFsOnDisk())
+	if err != nil {
+		t.Error(err)
+	}
+	executablePlugin := filepath.Join(
+		srcRoot, "someteam.example.com", "v1", "bashedconfigmap", "BashedConfigMap")
+	p := NewExecPlugin(executablePlugin)
+	err = p.ErrIfNotExecutable()
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	// Create a fake filesystem.
+	fSys := filesys.MakeFsInMemory()
+
+	// Load plugin config.
+	ldr, err := fLdr.NewLoader(
+		fLdr.RestrictionRootOnly, filesys.Separator, fSys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pvd := provider.NewDefaultDepProvider()
+	rf := resmap.NewFactory(pvd.GetResourceFactory())
+	pc := types.DisabledPluginConfig()
+
+	// Test for various lengths. 131071 is the max length that we can have for any given env var in Bytes.
+	tcs := []struct {
+		length int
+		char   rune
+	}{
+		{1000, 'a'},
+		{131071, 'a'},
+		{131072, 'a'},
+		{200000, 'a'},
+		{131071, '安'},
+		{131074, '安'},
+	}
+	for _, tc := range tcs {
+		t.Logf("Testing with an env var length of %d and character %c", tc.length, tc.char)
+		pluginConfig, err := rf.RF().FromBytes(buildLargePluginConfig(tc.length, tc.char))
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+		yaml, err := pluginConfig.AsYAML()
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+		err = p.Config(resmap.NewPluginHelpers(ldr, pvd.GetFieldValidator(), rf, pc), yaml)
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+		resMap, err := p.Generate()
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+		rNodeSlices := resMap.ToRNodeSlice()
+		for _, rNodeSlice := range rNodeSlices {
+			json, err := rNodeSlice.MarshalJSON()
+			if err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			if string(json) != expectedLargeConfigMap {
+				t.Fatalf("expected generated JSON to match %q, but got %q instead",
+					expectedLargeConfigMap, string(json))
+			}
+		}
+	}
+}
+
+// buildLargePluginConfig builds a plugin configuration of length: length - len("KUSTOMIZE_PLUGIN_CONFIG_STRING=")
+// This allows us to create an environment variable KUSTOMIZE_PLUGIN_CONFIG_STRING=<plugin content> with the exact
+// length that's provided in the length parameter. Used as a helper for TestExecPluginLarge.
+func buildLargePluginConfig(length int, char rune) []byte {
+	length -= len("KUSTOMIZE_PLUGIN_CONFIG_STRING=")
+
+	var sb strings.Builder
+	sb.WriteString("apiVersion: someteam.example.com/v1\n")
+	sb.WriteString("kind: BashedConfigMap\n")
+	sb.WriteString("metadata:\n")
+	sb.WriteString("  name: some-random-name\n")
+	sb.WriteString("argsOneLiner: \"user password\"\n")
+	sb.WriteString("customArg: ")
+
+	// Now, fill up parameter customArg: until we reach the desired length. Account for the fact that runes can be
+	// 1 to 4 Bytes each.
+	upperBound := length - sb.Len()
+	for i := 0; i < upperBound-len(string(char)); i += len(string(char)) {
+		sb.WriteRune(char)
+	}
+	return []byte(sb.String())
+}

api/internal/plugins/loader/load_go_plugin.go

@@ -0,0 +1,62 @@
+// Copyright 2024 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+//go:build !kustomize_disable_go_plugin_support
+
+package loader
+
+import (
+	"fmt"
+	"log"
+	"plugin"
+	"reflect"
+
+	"sigs.k8s.io/kustomize/api/internal/plugins/utils"
+	"sigs.k8s.io/kustomize/api/konfig"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/resid"
+)
+
+// registry is a means to avoid trying to load the same .so file
+// into memory more than once, which results in an error.
+// Each test makes its own loader, and tries to load its own plugins,
+// but the loaded .so files are in shared memory, so one will get
+// "this plugin already loaded" errors if the registry is maintained
+// as a Loader instance variable.  So make it a package variable.
+var registry = make(map[string]resmap.Configurable) //nolint:gochecknoglobals
+
+func copyPlugin(c resmap.Configurable) resmap.Configurable {
+	indirect := reflect.Indirect(reflect.ValueOf(c))
+	newIndirect := reflect.New(indirect.Type())
+	newIndirect.Elem().Set(reflect.ValueOf(indirect.Interface()))
+	newNamed := newIndirect.Interface()
+	return newNamed.(resmap.Configurable) //nolint:forcetypeassert
+}
+
+func (l *Loader) loadGoPlugin(id resid.ResId, absPath string) (resmap.Configurable, error) {
+	regId := relativePluginPath(id)
+	if c, ok := registry[regId]; ok {
+		return copyPlugin(c), nil
+	}
+	if !utils.FileExists(absPath) {
+		return nil, fmt.Errorf(
+			"expected file with Go object code at: %s", absPath)
+	}
+	log.Printf("Attempting plugin load from %q", absPath)
+	p, err := plugin.Open(absPath)
+	if err != nil {
+		return nil, errors.WrapPrefixf(err, "plugin %s fails to load", absPath)
+	}
+	symbol, err := p.Lookup(konfig.PluginSymbol)
+	if err != nil {
+		return nil, errors.WrapPrefixf(
+			err, "plugin %s doesn't have symbol %s",
+			regId, konfig.PluginSymbol)
+	}
+	c, ok := symbol.(resmap.Configurable)
+	if !ok {
+		return nil, fmt.Errorf("plugin %q not configurable", regId)
+	}
+	registry[regId] = c
+	return copyPlugin(c), nil
+}