feat: Release/v2.1.20

.github/workflows/sandbox.yaml

@@ -844,10 +844,10 @@ jobs:
           echo -e '```' >> $GITHUB_STEP_SUMMARY
 
   pr_creation:
-    if: startsWith(github.event.pull_request.head.ref, 'sandbox/')
     runs-on: ubuntu-latest
     steps:
     - name: Get a branch name if PR is created
+      if: startsWith(github.event.pull_request.head.ref, 'sandbox/')
       run: |
         # get a branch name
         branch_ref="${{ github.event.pull_request.head.ref }}"
@@ -859,6 +859,7 @@ jobs:
         echo "branch_ref=$branch_ref" >> $GITHUB_ENV
 
     - name: Render template
+      if: startsWith(github.event.pull_request.head.ref, 'sandbox/')
       id: template
       uses: chuhlomin/[email protected]
       with:
@@ -867,6 +868,7 @@ jobs:
           branch_ref: ${{ env.branch_ref }}
 
     - name: Create comment on a PR with the endpoints
+      if: startsWith(github.event.pull_request.head.ref, 'sandbox/')
       uses: peter-evans/create-or-update-comment@v4
       with:
         token: ${{ secrets.CI_BOT_TOKEN }}

api/v1/testkube.yaml

@@ -9451,6 +9451,10 @@ components:
           $ref: "#/components/schemas/BoxedInteger"
         multipleOf:
           $ref: "#/components/schemas/BoxedInteger"
+        sensitive:
+          type: boolean
+          default: false
+          description: whether this value should be stored in the secret
       required:
         - type
 

cmd/api-server/main.go

@@ -561,6 +561,19 @@ func main() {
 		exitOnError("Creating slack loader", err)
 	}
 
+	// TODO: Make granular environment variables, yet backwards compatible
+	secretConfig := testkube.SecretConfig{
+		Prefix:     cfg.SecretCreationPrefix,
+		List:       cfg.EnableSecretsEndpoint,
+		ListAll:    cfg.EnableSecretsEndpoint && cfg.EnableListingAllSecrets,
+		Create:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
+		Modify:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
+		Delete:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
+		AutoCreate: !cfg.DisableSecretCreation,
+	}
+
+	secretManager := secretmanager.New(clientset, secretConfig)
+
 	testWorkflowProcessor := presets.NewOpenSource(inspector)
 	if mode == common.ModeAgent {
 		testWorkflowProcessor = presets.NewPro(inspector)
@@ -576,6 +589,7 @@ func main() {
 		testWorkflowExecutionsClient,
 		testWorkflowsClient,
 		metrics,
+		secretManager,
 		serviceAccountNames,
 		cfg.GlobalWorkflowTemplateName,
 		cfg.TestkubeNamespace,
@@ -589,19 +603,6 @@ func main() {
 
 	go testWorkflowExecutor.Recover(context.Background())
 
-	// TODO: Make granular environment variables, yet backwards compatible
-	secretConfig := testkube.SecretConfig{
-		Prefix:     cfg.SecretCreationPrefix,
-		List:       cfg.EnableSecretsEndpoint,
-		ListAll:    cfg.EnableSecretsEndpoint && cfg.EnableListingAllSecrets,
-		Create:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
-		Modify:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
-		Delete:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
-		AutoCreate: !cfg.DisableSecretCreation,
-	}
-
-	secretManager := secretmanager.New(clientset, secretConfig)
-
 	api := apiv1.NewTestkubeAPI(
 		cfg.TestkubeNamespace,
 		resultsRepository,

cmd/tcl/testworkflow-toolkit/commands/parallel.go

@@ -33,6 +33,7 @@ import (
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/expressions"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowcontroller"
+	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/presets"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage"
@@ -202,7 +203,8 @@ func NewParallelCmd() *cobra.Command {
 				// Build the resources bundle
 				scheduledAt := time.Now()
 				bundle, err := presets.NewPro(inspector).
-					Bundle(context.Background(), &testworkflowsv1.TestWorkflow{Spec: *spec}, machine, baseMachine, params.MachineAt(index))
+					Bundle(context.Background(), &testworkflowsv1.TestWorkflow{Spec: *spec}, testworkflowprocessor.BundleOptions{},
+						machine, baseMachine, params.MachineAt(index))
 				if err != nil {
 					fmt.Printf("%d: failed to prepare resources: %s\n", index, err.Error())
 					registry.Destroy(index)

cmd/tcl/testworkflow-toolkit/commands/services.go

@@ -242,7 +242,8 @@ func NewServicesCmd() *cobra.Command {
 				// Build the resources bundle
 				scheduledAt := time.Now()
 				bundle, err := presets.NewPro(inspector).
-					Bundle(context.Background(), &testworkflowsv1.TestWorkflow{Spec: instance.Spec}, machine, baseMachine, params.MachineAt(index))
+					Bundle(context.Background(), &testworkflowsv1.TestWorkflow{Spec: instance.Spec}, testworkflowprocessor.BundleOptions{},
+						machine, baseMachine, params.MachineAt(index))
 				if err != nil {
 					log("error", "failed to build the service", err.Error())
 					return false

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/kubepug/kubepug v1.7.1
-	github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240819101144-e559df091f2b
+	github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d
 	github.com/minio/minio-go/v7 v7.0.47
 	github.com/montanaflynn/stats v0.6.6
 	github.com/moogar0880/problems v0.1.1

go.sum

@@ -371,8 +371,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubepug/kubepug v1.7.1 h1:LKhfSxS8Y5mXs50v+3Lpyec+cogErDLcV7CMUuiaisw=
 github.com/kubepug/kubepug v1.7.1/go.mod h1:lv+HxD0oTFL7ZWjj0u6HKhMbbTIId3eG7aWIW0gyF8g=
-github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240819101144-e559df091f2b h1:5DJ+7lhGTAp29loj1qbE36mzpG4xIFwsWQ7tNV4m8q8=
-github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240819101144-e559df091f2b/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
+github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d h1:ZwsRzKUJ+pMC2pF47L/O06Vq8vOYZe17GO3N2rL9lCk=
+github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=

internal/app/api/v1/testworkflows.go

@@ -327,8 +327,17 @@ func (s *TestkubeAPI) PreviewTestWorkflowHandler() fiber.Handler {
 				tplsMap[name] = *tpl
 			}
 
+			// Get information about execution namespace
+			// TODO: Think what to do when it is dynamic - create in all execution namespaces?
+			execNamespace := obj.Namespace
+			if obj.Spec.Job != nil && obj.Spec.Job.Namespace != "" {
+				execNamespace = obj.Spec.Job.Namespace
+			}
+
+			// Handle secrets auto-creation
+			secrets := s.SecretManager.Batch(execNamespace, "tw-", obj.Name)
 			// Resolve the TestWorkflow
-			err = testworkflowresolver.ApplyTemplates(obj, tplsMap)
+			err = testworkflowresolver.ApplyTemplates(obj, tplsMap, secrets.Append)
 			if err != nil {
 				return s.BadRequest(c, errPrefix, "resolving error", err)
 			}

pkg/api/v1/testkube/model_test_workflow_parameter_schema.go

@@ -29,4 +29,6 @@ type TestWorkflowParameterSchema struct {
 	ExclusiveMinimum *BoxedInteger `json:"exclusiveMinimum,omitempty"`
 	ExclusiveMaximum *BoxedInteger `json:"exclusiveMaximum,omitempty"`
 	MultipleOf       *BoxedInteger `json:"multipleOf,omitempty"`
+	// whether this value should be stored in the secret
+	Sensitive bool `json:"sensitive,omitempty"`
 }

pkg/expressions/generic.go

@@ -212,6 +212,11 @@ func resolve(v reflect.Value, t tagData, m []Machine, force bool, finalize bool)
 				vv, _ = expr2.Static().StringValue()
 			} else {
 				vv = expr.Template()
+				if t.value == "template" && !IsTemplateStringWithoutExpressions(str) {
+					if IsTemplateStringWithInternalFnCall(vv) {
+						vv = CleanTemplateStringInternalFnCall(vv)
+					}
+				}
 			}
 			changed = vv != str
 			if ptr.Kind() == reflect.String {

pkg/expressions/libs/secret.go

@@ -0,0 +1,44 @@
+package libs
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/kubeshop/testkube/pkg/expressions"
+)
+
+func NewSecretMachine(mapEnvs map[string]corev1.EnvVarSource) expressions.Machine {
+	return expressions.NewMachine().
+		RegisterFunction("secret", func(values ...expressions.StaticValue) (interface{}, bool, error) {
+			if len(values) != 2 {
+				return nil, true, fmt.Errorf(`"secret" function expects 2 arguments, %d provided`, len(values))
+			}
+
+			secretName, _ := values[0].StringValue()
+			keyName, _ := values[1].StringValue()
+			strs := []string{secretName, keyName}
+			for i := range strs {
+				j := 0
+				for _, char := range []string{"-", "."} {
+					for ; strings.Contains(strs[i], char); j++ {
+						strs[i] = strings.Replace(strs[i], char, fmt.Sprintf("_%d_", j), 1)
+					}
+				}
+			}
+
+			envName := fmt.Sprintf("S_N_%s_K_%s", strs[0], strs[1])
+			mapEnvs[envName] = corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: secretName,
+					},
+					Key: keyName,
+				},
+			}
+
+			return expressions.NewValue(fmt.Sprintf("{{%senv.%s}}", expressions.InternalFnCall, envName)), true, nil
+		})
+
+}

pkg/expressions/libs/secret_test.go

@@ -0,0 +1,26 @@
+package libs
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/kubeshop/testkube/pkg/expressions"
+)
+
+func TestSecret(t *testing.T) {
+	mapEnvs := make(map[string]corev1.EnvVarSource)
+	machine := NewSecretMachine(mapEnvs)
+	assert.Equal(t, "{{"+expressions.InternalFnCall+"env.S_N_name_0_one_1_two_K_key_0_three_1_four}}", MustCall(machine, "secret", "name-one.two", "key-three.four"))
+	assert.EqualValues(t, map[string]corev1.EnvVarSource{
+		"S_N_name_0_one_1_two_K_key_0_three_1_four": {
+			SecretKeyRef: &corev1.SecretKeySelector{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: "name-one.two",
+				},
+				Key: "key-three.four",
+			},
+		},
+	}, mapEnvs)
+}

pkg/expressions/parse.go

@@ -255,3 +255,11 @@ func CompileAndResolveTemplate(tpl string, m ...Machine) (Expression, error) {
 func IsTemplateStringWithoutExpressions(tpl string) bool {
 	return !strings.Contains(tpl, "{{")
 }
+
+func IsTemplateStringWithInternalFnCall(tpl string) bool {
+	return strings.Contains(tpl, "{{\"{{\"}}"+InternalFnCall)
+}
+
+func CleanTemplateStringInternalFnCall(tpl string) string {
+	return strings.ReplaceAll(tpl, "{{\"{{\"}}"+InternalFnCall, "{{")
+}

pkg/expressions/stdlib.go

@@ -17,7 +17,8 @@ import (
 )
 
 const (
-	RFC3339Millis = "2006-01-02T15:04:05.000Z07:00"
+	RFC3339Millis  = "2006-01-02T15:04:05.000Z07:00"
+	InternalFnCall = "__internal__fn__call__"
 )
 
 type StdFunction struct {

pkg/mapper/testworkflows/kube_openapi.go

@@ -370,6 +370,7 @@ func MapParameterSchemaKubeToAPI(v testworkflowsv1.ParameterSchema) testkube.Tes
 		ExclusiveMinimum: MapInt64ToBoxedInteger(v.ExclusiveMinimum),
 		ExclusiveMaximum: MapInt64ToBoxedInteger(v.ExclusiveMaximum),
 		MultipleOf:       MapInt64ToBoxedInteger(v.MultipleOf),
+		Sensitive:        v.Sensitive,
 	}
 }
 

pkg/mapper/testworkflows/openapi_kube.go

@@ -269,6 +269,7 @@ func MapParameterSchemaAPIToKube(v testkube.TestWorkflowParameterSchema) testwor
 			ExclusiveMaximum: MapBoxedIntegerToInt64(v.ExclusiveMaximum),
 			MultipleOf:       MapBoxedIntegerToInt64(v.MultipleOf),
 		},
+		Sensitive: v.Sensitive,
 	}
 }
 

pkg/secretmanager/batch.go

@@ -117,3 +117,7 @@ func (s *batch) Create(ctx context.Context, owner *metav1.OwnerReference) error
 	}
 	return nil
 }
+
+func (s *batch) Get() []corev1.Secret {
+	return s.secrets
+}