feat: alternative temp directory for generated manifests

Signed-off-by: Jonathan West <[email protected]>

Author: jgwest

pkg/cache/cluster.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/klog/v2/textlogger"
 	"k8s.io/kubectl/pkg/util/openapi"
 
+	"github.com/argoproj/gitops-engine/pkg/utils/io"
 	"github.com/argoproj/gitops-engine/pkg/utils/kube"
 	"github.com/argoproj/gitops-engine/pkg/utils/tracing"
 )
@@ -155,8 +156,9 @@ func NewClusterCache(config *rest.Config, opts ...UpdateSettingsFunc) *clusterCa
 		nsIndex:            make(map[string]map[kube.ResourceKey]*Resource),
 		config:             config,
 		kubectl: &kube.KubectlCmd{
-			Log:    log,
-			Tracer: tracing.NopTracer{},
+			Log:     log,
+			Tracer:  tracing.NopTracer{},
+			TmpPath: io.TempPathUseDevShmIfAvailable(),
 		},
 		syncStatus: clusterCacheSync{
 			resyncTimeout: defaultClusterResyncTimeout,

pkg/cache/settings.go

@@ -170,3 +170,12 @@ func SetRespectRBAC(respectRBAC int) UpdateSettingsFunc {
 		}
 	}
 }
+
+// SetTmpPath sets the path used to store temporary files that are passed to kubectl code. These temporary files are usually cluster credentials or kubernetes manifests. See 'utils/io/io.go' for details.
+func SetTmpPath(tmpPath string) UpdateSettingsFunc {
+	return func(cache *clusterCache) {
+		if kcmd, ok := cache.kubectl.(*kube.KubectlCmd); ok {
+			kcmd.TmpPath = tmpPath
+		}
+	}
+}

pkg/engine/engine_options.go

@@ -4,6 +4,7 @@ import (
 	"github.com/go-logr/logr"
 	"k8s.io/klog/v2/textlogger"
 
+	"github.com/argoproj/gitops-engine/pkg/utils/io"
 	"github.com/argoproj/gitops-engine/pkg/utils/kube"
 	"github.com/argoproj/gitops-engine/pkg/utils/tracing"
 )
@@ -20,8 +21,9 @@ func applyOptions(opts []Option) options {
 	o := options{
 		log: log,
 		kubectl: &kube.KubectlCmd{
-			Log:    log,
-			Tracer: tracing.NopTracer{},
+			Log:     log,
+			Tracer:  tracing.NopTracer{},
+			TmpPath: io.TempPathUseDevShmIfAvailable(),
 		},
 	}
 	for _, opt := range opts {

pkg/utils/io/io.go

@@ -1,21 +1,31 @@
 package io
 
-import (
-	"os"
-)
+import "os"
 
 var (
-	// TempDir is set to '/dev/shm' if exists, otherwise is "", which defaults to os.TempDir() when passed to os.CreateTemp()
-	TempDir string
+	// TempDir is set to '/dev/shm' if it exists, otherwise is "", which defaults to os.TempDir() when passed to os.CreateTemp()
+	devShmTempPath string
 )
 
 func init() {
 	fileInfo, err := os.Stat("/dev/shm")
 	if err == nil && fileInfo.IsDir() {
-		TempDir = "/dev/shm"
+		devShmTempPath = "/dev/shm"
 	}
 }
 
+// TempPathUseDevShmIfAvailable will return '/dev/shm' if it is available on the system, otherwise it will return "", which defaults to os.TempDir() when passed to os.CreateTemp()
+//
+// The result of this function is used to store temporary files that are passed to kubectl code. These temporary files are usually cluster credentials or kubernetes manifests.
+//
+// NOTE: There are tradeoffs to using this function: '/dev/shm' is backed by RAM, and thus has limited size.
+// - Since it is backed by RAM, this has the advantage of ensuring that sensitive data (such as credentials) are kept off disk (absent disk caching of memory)
+// - However, due to the limited size, '/dev/shm' may run out of disk space, and/or is more vulnerable to slow leaks of files over time.
+// You may instead consider using a disk-backed storage path like "", which os.CreateTemp() will default to e.g. '/tmp'.
+func TempPathUseDevShmIfAvailable() string {
+	return devShmTempPath
+}
+
 // DeleteFile is best effort deletion of a file
 func DeleteFile(path string) {
 	if _, err := os.Stat(path); os.IsNotExist(err) {

pkg/utils/kube/ctl.go

@@ -45,6 +45,8 @@ type KubectlCmd struct {
 	Log          logr.Logger
 	Tracer       tracing.Tracer
 	OnKubectlRun OnKubectlRunFunc
+	// TmpPath is used to store temporary files that are passed to kubectl code. These temporary files are usually cluster credentials or kubernetes manifests. See 'utils/io/io.go' for details.
+	TmpPath string
 }
 
 type APIResourceInfo struct {
@@ -272,7 +274,7 @@ func (k *KubectlCmd) DeleteResource(ctx context.Context, config *rest.Config, gv
 }
 
 func (k *KubectlCmd) ManageResources(config *rest.Config, openAPISchema openapi.Resources) (ResourceOperations, func(), error) {
-	f, err := os.CreateTemp(utils.TempDir, "")
+	f, err := os.CreateTemp(k.TmpPath, "")
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to generate temp file for kubeconfig: %v", err)
 	}
@@ -293,6 +295,7 @@ func (k *KubectlCmd) ManageResources(config *rest.Config, openAPISchema openapi.
 		tracer:        k.Tracer,
 		log:           k.Log,
 		onKubectlRun:  k.OnKubectlRun,
+		tmpPath:       k.TmpPath,
 	}, cleanup, nil
 }
 

pkg/utils/kube/ctl_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"k8s.io/klog/v2/textlogger"
 
+	"github.com/argoproj/gitops-engine/pkg/utils/io"
 	testingutils "github.com/argoproj/gitops-engine/pkg/utils/testing"
 	"github.com/argoproj/gitops-engine/pkg/utils/tracing"
 )
@@ -20,8 +21,9 @@ var (
 
 func TestConvertToVersion(t *testing.T) {
 	kubectl := KubectlCmd{
-		Log:    textlogger.NewLogger(textlogger.NewConfig()),
-		Tracer: tracing.NopTracer{},
+		Log:     textlogger.NewLogger(textlogger.NewConfig()),
+		Tracer:  tracing.NopTracer{},
+		TmpPath: io.TempPathUseDevShmIfAvailable(),
 	}
 	t.Run("AppsDeployment", func(t *testing.T) {
 		newObj, err := kubectl.ConvertToVersion(testingutils.UnstructuredFromFile("testdata/appsdeployment.yaml"), "apps", "v1")

pkg/utils/kube/resource_ops.go

@@ -52,6 +52,8 @@ type kubectlResourceOperations struct {
 	onKubectlRun  OnKubectlRunFunc
 	fact          cmdutil.Factory
 	openAPISchema openapi.Resources
+	// tmpPath is used to store temporary files that are passed to kubectl code. These temporary files are usually cluster credentials or kubernetes manifests. See 'utils/io/io.go' for details.
+	tmpPath string
 }
 
 type commandExecutor func(f cmdutil.Factory, ioStreams genericclioptions.IOStreams, fileName string) error
@@ -61,16 +63,16 @@ func (k *kubectlResourceOperations) runResourceCommand(ctx context.Context, obj
 	if err != nil {
 		return "", err
 	}
-	manifestFile, err := os.CreateTemp(io.TempDir, "")
+	manifestFile, err := os.CreateTemp(k.tmpPath, "")
 	if err != nil {
-		return "", fmt.Errorf("Failed to generate temp file for manifest: %v", err)
+		return "", fmt.Errorf("failed to generate temp file for manifest: %v", err)
 	}
 	defer io.DeleteFile(manifestFile.Name())
 	if _, err = manifestFile.Write(manifestBytes); err != nil {
-		return "", fmt.Errorf("Failed to write manifest: %v", err)
+		return "", fmt.Errorf("failed to write manifest: %v", err)
 	}
 	if err = manifestFile.Close(); err != nil {
-		return "", fmt.Errorf("Failed to close manifest: %v", err)
+		return "", fmt.Errorf("failed to close manifest: %v", err)
 	}
 
 	// log manifest