diff --git a/tooling/templatize/pkg/config/types_test.go b/tooling/templatize/pkg/config/types_test.go new file mode 100644 index 000000000..8285e3734 --- /dev/null +++ b/tooling/templatize/pkg/config/types_test.go @@ -0,0 +1,45 @@ +package config + +import "testing" + +func TestGetByPath(t *testing.T) { + tests := []struct { + name string + vars Variables + path string + want any + found bool + }{ + { + name: "simple", + vars: Variables{ + "key": "value", + }, + path: "key", + want: "value", + found: true, + }, + { + name: "nested", + vars: Variables{ + "key": Variables{ + "key": "value", + }, + }, + path: "key.key", + want: "value", + found: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, found := tt.vars.GetByPath(tt.path) + if got != tt.want { + t.Errorf("Variables.GetByPath() got = %v, want %v", got, tt.want) + } + if found != tt.found { + t.Errorf("Variables.GetByPath() found = %v, want %v", found, tt.found) + } + }) + } +} diff --git a/tooling/templatize/pkg/ev2/pipeline_test.go b/tooling/templatize/pkg/ev2/pipeline_test.go index 4a0c0f983..540476335 100644 --- a/tooling/templatize/pkg/ev2/pipeline_test.go +++ b/tooling/templatize/pkg/ev2/pipeline_test.go @@ -31,8 +31,10 @@ func TestPrecompilePipelineForEV2(t *testing.T) { } fmt.Println(p) expectedParamsPath := "ev2-precompiled-test.bicepparam" - if p.ResourceGroups[0].Steps[1].Parameters != expectedParamsPath { - t.Errorf("expected parameters path %v, but got %v", expectedParamsPath, p.ResourceGroups[0].Steps[1].Parameters) + + armStep := p.ResourceGroups[0].Steps[2] + if armStep.Parameters != expectedParamsPath { + t.Errorf("expected parameters path %v, but got %v", expectedParamsPath, armStep.Parameters) } // TODO improve test, check against fixture } diff --git a/tooling/templatize/pkg/pipeline/inspect.go b/tooling/templatize/pkg/pipeline/inspect.go index aaadcae5a..739b7c50b 100644 --- a/tooling/templatize/pkg/pipeline/inspect.go +++ b/tooling/templatize/pkg/pipeline/inspect.go @@ -49,7 +49,7 @@ func inspectVars(s *step, options *PipelineInspectOptions, writer io.Writer) err var err error switch s.Action { case "Shell": - envVars, err = s.getEnvVars(options.Vars, false) + envVars, err = s.mapStepVariables(options.Vars) default: return fmt.Errorf("inspecting step variables not implemented for action type %s", s.Action) } diff --git a/tooling/templatize/pkg/pipeline/run.go b/tooling/templatize/pkg/pipeline/run.go index 783e1eae1..c8fb29fac 100644 --- a/tooling/templatize/pkg/pipeline/run.go +++ b/tooling/templatize/pkg/pipeline/run.go @@ -93,6 +93,19 @@ func (rg *resourceGroup) run(ctx context.Context, options *PipelineRunOptions) e } logger := logr.FromContextOrDiscard(ctx) + + kubeconfigFile, err := prepareKubeConfig(ctx, executionTarget) + if kubeconfigFile != "" { + defer func() { + if err := os.Remove(kubeconfigFile); err != nil { + logger.V(5).Error(err, "failed to delete kubeconfig file", "kubeconfig", kubeconfigFile) + } + }() + } + if err != nil { + return fmt.Errorf("failed to prepare kubeconfig: %w", err) + } + for _, step := range rg.Steps { if options.Step != "" && step.Name != options.Step { // skip steps that don't match the specified step name @@ -109,6 +122,7 @@ func (rg *resourceGroup) run(ctx context.Context, options *PipelineRunOptions) e "aksCluster", executionTarget.AKSClusterName, ), ), + kubeconfigFile, executionTarget, options, ) if err != nil { @@ -118,13 +132,17 @@ func (rg *resourceGroup) run(ctx context.Context, options *PipelineRunOptions) e return nil } -func (s *step) run(ctx context.Context, executionTarget *ExecutionTarget, options *PipelineRunOptions) error { +func (s *step) run(ctx context.Context, kubeconfigFile string, executionTarget *ExecutionTarget, options *PipelineRunOptions) error { fmt.Println("\n---------------------") + if options.DryRun { + fmt.Println("This is a dry run!") + } fmt.Println(s.description()) fmt.Print("\n") + switch s.Action { case "Shell": - return s.runShellStep(ctx, executionTarget, options) + return s.runShellStep(ctx, kubeconfigFile, options) case "ARM": return s.runArmStep(ctx, executionTarget, options) default: @@ -132,6 +150,25 @@ func (s *step) run(ctx context.Context, executionTarget *ExecutionTarget, option } } +func prepareKubeConfig(ctx context.Context, executionTarget *ExecutionTarget) (string, error) { + logger := logr.FromContextOrDiscard(ctx) + kubeconfigFile := "" + if executionTarget.AKSClusterName != "" { + logger.V(5).Info("Building kubeconfig for AKS cluster") + kubeconfigFile, err := executionTarget.KubeConfig(ctx) + if err != nil { + return "", fmt.Errorf("failed to build kubeconfig for %s: %w", executionTarget.aksID(), err) + } + defer func() { + if err := os.Remove(kubeconfigFile); err != nil { + logger.V(5).Error(err, "failed to delete kubeconfig file", "kubeconfig", kubeconfigFile) + } + }() + logger.V(5).Info("kubeconfig set to shell execution environment", "kubeconfig", kubeconfigFile) + } + return kubeconfigFile, nil +} + func (s *step) description() string { var details []string switch s.Action { diff --git a/tooling/templatize/pkg/pipeline/shell.go b/tooling/templatize/pkg/pipeline/shell.go index bac670e7f..23754a119 100644 --- a/tooling/templatize/pkg/pipeline/shell.go +++ b/tooling/templatize/pkg/pipeline/shell.go @@ -3,7 +3,7 @@ package pipeline import ( "context" "fmt" - "os" + "maps" "os/exec" "github.com/go-logr/logr" @@ -12,75 +12,75 @@ import ( "github.com/Azure/ARO-HCP/tooling/templatize/pkg/utils" ) -func (s *step) runShellStep(ctx context.Context, executionTarget *ExecutionTarget, options *PipelineRunOptions) error { +func (s *step) createCommand(ctx context.Context, dryRun bool, envVars map[string]string) (*exec.Cmd, bool) { + var cmd *exec.Cmd + if dryRun { + if s.DryRun.Command == nil && s.DryRun.EnvVars == nil { + return nil, true + } + for _, e := range s.DryRun.EnvVars { + envVars[e.Name] = e.Value + } + if s.DryRun.Command != nil { + cmd = exec.CommandContext(ctx, s.DryRun.Command[0], s.DryRun.Command[1:]...) + } + } + if cmd == nil { + // if dry-run is not enabled, use the actual command or also if no dry-run command is defined + cmd = exec.CommandContext(ctx, s.Command[0], s.Command[1:]...) + } + cmd.Env = append(cmd.Env, utils.MapToEnvVarArray(envVars)...) + return cmd, false +} + +func (s *step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions) error { + if s.outputFunc == nil { + s.outputFunc = func(output string) { + fmt.Println(output) + } + } + logger := logr.FromContextOrDiscard(ctx) // build ENV vars - envVars, err := s.getEnvVars(options.Vars, true) + stepVars, err := s.mapStepVariables(options.Vars) if err != nil { return fmt.Errorf("failed to build env vars: %w", err) } - // prepare kubeconfig - if executionTarget.AKSClusterName != "" { - logger.V(5).Info("Building kubeconfig for AKS cluster") - kubeconfigFile, err := executionTarget.KubeConfig(ctx) - if err != nil { - return fmt.Errorf("failed to build kubeconfig for %s: %w", executionTarget.aksID(), err) - } - defer func() { - if err := os.Remove(kubeconfigFile); err != nil { - logger.V(5).Error(err, "failed to delete kubeconfig file", "kubeconfig", kubeconfigFile) - } - }() - envVars["KUBECONFIG"] = kubeconfigFile - logger.V(5).Info("kubeconfig set to shell execution environment", "kubeconfig", kubeconfigFile) + envVars := utils.GetOsVariable() + + maps.Copy(envVars, stepVars) + // execute the command + cmd, skipCommand := s.createCommand(ctx, options.DryRun, envVars) + if skipCommand { + logger.V(5).Info("Skipping step '%s' due to missing dry-run configuiration", s.Name) + return nil } - // TODO handle dry-run + if kubeconfigFile != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("KUBECONFIG=%s", kubeconfigFile)) + } - // execute the command logger.V(5).Info(fmt.Sprintf("Executing shell command: %s\n", s.Command), "command", s.Command) - cmd := exec.CommandContext(ctx, s.Command[0], s.Command[1:]...) - cmd.Env = append(cmd.Env, utils.MapToEnvVarArray(envVars)...) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to execute shell command: %s %w", string(output), err) } - // print the output of the command - fmt.Println(string(output)) + s.outputFunc(string(output)) return nil } -func (s *step) getEnvVars(vars config.Variables, includeOSEnvVars bool) (map[string]string, error) { +func (s *step) mapStepVariables(vars config.Variables) (map[string]string, error) { envVars := make(map[string]string) - envVars["RUNS_IN_TEMPLATIZE"] = "1" - if includeOSEnvVars { - for k, v := range utils.GetOSEnvVarsAsMap() { - envVars[k] = v - } - } for _, e := range s.Env { value, found := vars.GetByPath(e.ConfigRef) if !found { return nil, fmt.Errorf("failed to lookup config reference %s for %s", e.ConfigRef, e.Name) } - envVars[e.Name] = anyToString(value) + envVars[e.Name] = utils.AnyToString(value) } return envVars, nil } - -func anyToString(value any) string { - switch v := value.(type) { - case string: - return v - case int: - return fmt.Sprintf("%d", v) - case bool: - return fmt.Sprintf("%t", v) - default: - return fmt.Sprintf("%v", v) - } -} diff --git a/tooling/templatize/pkg/pipeline/shell_test.go b/tooling/templatize/pkg/pipeline/shell_test.go new file mode 100644 index 000000000..4ac7f6a6e --- /dev/null +++ b/tooling/templatize/pkg/pipeline/shell_test.go @@ -0,0 +1,168 @@ +package pipeline + +import ( + "context" + "testing" + + "gotest.tools/v3/assert" + + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" +) + +func TestCreateCommand(t *testing.T) { + ctx := context.Background() + testCases := []struct { + name string + step *step + dryRun bool + envVars map[string]string + expectedCommand string + expectedArgs []string + expectedEnv []string + skipCommand bool + }{ + { + name: "basic", + step: &step{ + Command: []string{"/usr/bin/echo", "hello"}, + }, + expectedCommand: "/usr/bin/echo", + expectedArgs: []string{"hello"}, + }, + { + name: "dry-run", + step: &step{ + Command: []string{"/usr/bin/echo", "hello"}, + DryRun: dryRun{ + Command: []string{"/usr/bin/echo", "dry-run"}, + }, + }, + dryRun: true, + expectedCommand: "/usr/bin/echo", + expectedArgs: []string{"dry-run"}, + }, + { + name: "dry-run-env", + step: &step{ + Command: []string{"/usr/bin/echo"}, + DryRun: dryRun{ + EnvVars: []EnvVar{ + { + Name: "DRY_RUN", + Value: "true", + }, + }, + }, + }, + dryRun: true, + expectedCommand: "/usr/bin/echo", + envVars: map[string]string{}, + expectedEnv: []string{"DRY_RUN=true"}, + }, + { + name: "dry-run fail", + step: &step{ + Command: []string{"/usr/bin/echo"}, + }, + dryRun: true, + skipCommand: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cmd, skipCommand := tc.step.createCommand(ctx, tc.dryRun, tc.envVars) + assert.Equal(t, skipCommand, tc.skipCommand) + if !tc.skipCommand { + assert.Equal(t, cmd.Path, tc.expectedCommand) + } + if tc.expectedArgs != nil { + assert.DeepEqual(t, cmd.Args[1:], tc.expectedArgs) + } + if tc.expectedEnv != nil { + assert.DeepEqual(t, cmd.Env, tc.expectedEnv) + } + }) + } + +} + +func TestMapStepVariables(t *testing.T) { + testCases := []struct { + name string + vars config.Variables + step step + expected map[string]string + err string + }{ + { + name: "basic", + vars: config.Variables{ + "FOO": "bar", + }, + step: step{ + Env: []EnvVar{ + { + Name: "BAZ", + ConfigRef: "FOO", + }, + }, + }, + expected: map[string]string{ + "BAZ": "bar", + }, + }, + { + name: "missing", + vars: config.Variables{}, + step: step{ + Env: []EnvVar{ + { + ConfigRef: "FOO", + }, + }, + }, + err: "failed to lookup config reference FOO for ", + }, + { + name: "type conversion", + vars: config.Variables{ + "FOO": 42, + }, + step: step{ + Env: []EnvVar{ + { + Name: "BAZ", + ConfigRef: "FOO", + }, + }, + }, + expected: map[string]string{ + "BAZ": "42", + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + envVars, err := tc.step.mapStepVariables(tc.vars) + t.Log(envVars) + if tc.err != "" { + assert.Error(t, err, tc.err) + } else { + assert.NilError(t, err) + assert.DeepEqual(t, envVars, tc.expected) + } + }) + } +} + +func TestRunShellStep(t *testing.T) { + expectedOutput := "hello\n" + s := &step{ + Command: []string{"echo", "hello"}, + outputFunc: func(output string) { + assert.Equal(t, output, expectedOutput) + }, + } + err := s.runShellStep(context.Background(), "", &PipelineRunOptions{}) + assert.NilError(t, err) +} diff --git a/tooling/templatize/pkg/pipeline/types.go b/tooling/templatize/pkg/pipeline/types.go index 58db883f5..4dc9fb3df 100644 --- a/tooling/templatize/pkg/pipeline/types.go +++ b/tooling/templatize/pkg/pipeline/types.go @@ -14,6 +14,8 @@ type resourceGroup struct { Steps []*step `yaml:"steps"` } +type outPutHandler func(string) + type step struct { Name string `yaml:"name"` Action string `yaml:"action"` @@ -22,9 +24,17 @@ type step struct { Template string `yaml:"template"` Parameters string `yaml:"parameters"` DependsOn []string `yaml:"dependsOn"` + DryRun dryRun `yaml:"dryRun"` + outputFunc outPutHandler +} + +type dryRun struct { + EnvVars []EnvVar `yaml:"envVars"` + Command []string `yaml:"command"` } type EnvVar struct { Name string `yaml:"name"` ConfigRef string `yaml:"configRef"` + Value string `yaml:"value"` } diff --git a/tooling/templatize/pkg/utils/env.go b/tooling/templatize/pkg/utils/env.go index 84c51b537..2798a3746 100644 --- a/tooling/templatize/pkg/utils/env.go +++ b/tooling/templatize/pkg/utils/env.go @@ -2,21 +2,30 @@ package utils import ( "fmt" + "maps" "os" "strings" ) -func GetOSEnvVarsAsMap() map[string]string { +// GetOsVariable looks up OS environment variables and returns them as a map. +// It also sets a special environment variable RUNS_IN_TEMPLATIZE to 1. +func GetOsVariable() map[string]string { envVars := make(map[string]string) + envVars["RUNS_IN_TEMPLATIZE"] = "1" + + osVars := make(map[string]string) for _, env := range os.Environ() { parts := strings.SplitN(env, "=", 2) if len(parts) == 2 { envVars[parts[0]] = parts[1] } } + maps.Copy(envVars, osVars) + return envVars } +// MapToEnvVarArray converts a map of environment variables to an array of strings. func MapToEnvVarArray(envVars map[string]string) []string { envVarArray := make([]string, 0, len(envVars)) for k, v := range envVars { diff --git a/tooling/templatize/pkg/utils/env_test.go b/tooling/templatize/pkg/utils/env_test.go new file mode 100644 index 000000000..2d15ac8c0 --- /dev/null +++ b/tooling/templatize/pkg/utils/env_test.go @@ -0,0 +1,22 @@ +package utils + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestGetOsVariable(t *testing.T) { + t.Setenv("FOO", "BAR") + envVars := GetOsVariable() + assert.Equal(t, "1", envVars["RUNS_IN_TEMPLATIZE"]) + assert.Equal(t, "BAR", envVars["FOO"]) +} + +func TestMapToEnvVarArray(t *testing.T) { + envVars := map[string]string{ + "FOO": "BAR", + } + envVarArray := MapToEnvVarArray(envVars) + assert.DeepEqual(t, []string{"FOO=BAR"}, envVarArray) +} diff --git a/tooling/templatize/pkg/utils/typing.go b/tooling/templatize/pkg/utils/typing.go new file mode 100644 index 000000000..dba9a4f49 --- /dev/null +++ b/tooling/templatize/pkg/utils/typing.go @@ -0,0 +1,19 @@ +package utils + +import ( + "fmt" +) + +// AnyToString maps some types to strings, as they are used in OS Env. +func AnyToString(value any) string { + switch v := value.(type) { + case string: + return v + case int: + return fmt.Sprintf("%d", v) + case bool: + return fmt.Sprintf("%t", v) + default: + return fmt.Sprintf("%v", v) + } +} diff --git a/tooling/templatize/pkg/utils/typing_test.go b/tooling/templatize/pkg/utils/typing_test.go new file mode 100644 index 000000000..9797c1455 --- /dev/null +++ b/tooling/templatize/pkg/utils/typing_test.go @@ -0,0 +1,14 @@ +package utils + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestAnyToString(t *testing.T) { + assert.Equal(t, "foo", AnyToString("foo")) + assert.Equal(t, "42", AnyToString(42)) + assert.Equal(t, "true", AnyToString(true)) + assert.Equal(t, "3.14", AnyToString(3.14)) +} diff --git a/tooling/templatize/testdata/Makefile b/tooling/templatize/testdata/Makefile new file mode 100644 index 000000000..71375ec54 --- /dev/null +++ b/tooling/templatize/testdata/Makefile @@ -0,0 +1,4 @@ +deploy: + echo ${DRY_RUN} + +.PHONE: deploy diff --git a/tooling/templatize/testdata/pipeline.yaml b/tooling/templatize/testdata/pipeline.yaml index 702bb39d5..bd61629dc 100644 --- a/tooling/templatize/testdata/pipeline.yaml +++ b/tooling/templatize/testdata/pipeline.yaml @@ -11,6 +11,13 @@ resourceGroups: env: - name: MAESTRO_IMAGE configRef: maestro_image + - name: dry-run + action: Shell + command: ["make", "deploy"] + dryRun: + envVars: + - name: DRY_RUN + value: "A very dry one" - name: svc action: ARM template: templates/svc-cluster.bicep diff --git a/tooling/templatize/testdata/zz_fixture_TestPreprocessFileForEV2SystemVars.yaml b/tooling/templatize/testdata/zz_fixture_TestPreprocessFileForEV2SystemVars.yaml index 9540159a7..b243c236c 100644 --- a/tooling/templatize/testdata/zz_fixture_TestPreprocessFileForEV2SystemVars.yaml +++ b/tooling/templatize/testdata/zz_fixture_TestPreprocessFileForEV2SystemVars.yaml @@ -11,6 +11,13 @@ resourceGroups: env: - name: MAESTRO_IMAGE configRef: maestro_image + - name: dry-run + action: Shell + command: ["make", "deploy"] + dryRun: + envVars: + - name: DRY_RUN + value: "A very dry one" - name: svc action: ARM template: templates/svc-cluster.bicep diff --git a/tooling/templatize/testdata/zz_fixture_TestRawOptions.yaml b/tooling/templatize/testdata/zz_fixture_TestRawOptions.yaml index fc58ba012..4b8ed8ef3 100644 --- a/tooling/templatize/testdata/zz_fixture_TestRawOptions.yaml +++ b/tooling/templatize/testdata/zz_fixture_TestRawOptions.yaml @@ -11,6 +11,13 @@ resourceGroups: env: - name: MAESTRO_IMAGE configRef: maestro_image + - name: dry-run + action: Shell + command: ["make", "deploy"] + dryRun: + envVars: + - name: DRY_RUN + value: "A very dry one" - name: svc action: ARM template: templates/svc-cluster.bicep