refactor test for templatize ev2 package

* `PrecompilePipelineForEV2` renamed to `PrecompilePipelineFileForEV2`
* `PrecompilePipelineForEV2` reintroduced but returns a parsed `pipeline.Pipeline` struct
* refactoring for testability

Signed-off-by: Gerd Oberlechner <[email protected]>

tooling/templatize/pkg/config/config.go

@@ -247,20 +247,28 @@ func (cp *configProviderImpl) loadConfig(configReplacements *ConfigReplacements)
 // PreprocessFile reads and processes a gotemplate
 // The path will be read as is. It parses the file as a template, and executes it with the provided variables.
 func PreprocessFile(templateFilePath string, vars map[string]any) ([]byte, error) {
-	tmpl := template.New("file")
 	content, err := os.ReadFile(templateFilePath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read file %s: %w", templateFilePath, err)
 	}
+	processedContent, err := PreprocessContent(content, vars)
+	if err != nil {
+		return nil, fmt.Errorf("failed to preprocess file %s: %w", templateFilePath, err)
+	}
+	return processedContent, nil
+}
 
-	tmpl, err = tmpl.Parse(string(content))
+// PreprocessFile processes a gotemplate from memory
+func PreprocessContent(content []byte, vars map[string]any) ([]byte, error) {
+	tmpl := template.New("file")
+	tmpl, err := tmpl.Parse(string(content))
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse template %s: %w", templateFilePath, err)
+		return nil, fmt.Errorf("failed to parse template: %w", err)
 	}
 
 	var tmplBytes bytes.Buffer
 	if err := tmpl.Option("missingkey=error").Execute(&tmplBytes, vars); err != nil {
-		return nil, fmt.Errorf("failed to execute template %s: %w", templateFilePath, err)
+		return nil, fmt.Errorf("failed to execute template: %w", err)
 	}
 	return tmplBytes.Bytes(), nil
 }

tooling/templatize/pkg/ev2/pipeline.go

@@ -1,6 +1,7 @@
 package ev2
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -10,73 +11,109 @@ import (
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
 
-func PrecompilePipelineForEV2(pipelineFilePath string, vars config.Variables) (string, error) {
-	// switch to the pipeline file dir so all relative paths are resolved correctly
-	originalDir, err := os.Getwd()
-	if err != nil {
-		return "", nil
-	}
-	pipelineDir := filepath.Dir(pipelineFilePath)
-	err = os.Chdir(pipelineDir)
+const precompiledPrefix = "ev2-precompiled-"
+
+func PrecompilePipelineFileForEV2(pipelineFilePath string, vars config.Variables) (string, error) {
+	precompiledPipeline, err := PrecompilePipelineForEV2(pipelineFilePath, vars)
 	if err != nil {
 		return "", err
 	}
-	defer func() {
-		_ = os.Chdir(originalDir)
-	}()
 
-	// precompile the pipeline file
-	pipelineFileName := filepath.Base(pipelineFilePath)
-	p, err := pipeline.NewPipelineFromFile(pipelineFileName, vars)
+	// store as new file
+	pipelineBytes, err := yaml.Marshal(precompiledPipeline)
 	if err != nil {
 		return "", err
 	}
-	err = processPipelineForEV2(p, vars)
+	err = os.WriteFile(precompiledPipeline.PipelineFilePath(), pipelineBytes, 0644)
 	if err != nil {
 		return "", err
 	}
 
-	// store as new file
-	pipelineBytes, err := yaml.Marshal(p)
+	return precompiledPipeline.PipelineFilePath(), nil
+}
+
+func PrecompilePipelineForEV2(pipelineFilePath string, vars config.Variables) (*pipeline.Pipeline, error) {
+	// load the pipeline and referenced files
+	originalPipeline, err := pipeline.NewPipelineFromFile(pipelineFilePath, vars)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	newPipelineFileName := "ev2-precompiled-" + pipelineFileName
-	err = os.WriteFile(newPipelineFileName, pipelineBytes, 0644)
+	referencedFiles, err := readReferencedPipelineFiles(originalPipeline)
 	if err != nil {
-		return "", err
+		return nil, fmt.Errorf("failed to read referenced files of pipeline %s: %w", originalPipeline.PipelineFilePath(), err)
+	}
+
+	// precompile the pipeline and referenced files
+	processedPipeline, processedFiles, err := processPipelineForEV2(originalPipeline, referencedFiles, vars)
+	if err != nil {
+		return nil, err
+	}
+
+	// store the processed files to disk relative to the pipeline directory
+	_, restoreDir, err := processedPipeline.EnterPipelineDir()
+	if err != nil {
+		return nil, fmt.Errorf("failed to enter pipeline directory: %w", err)
+	}
+	defer restoreDir()
+	for filePath, content := range processedFiles {
+		err := os.WriteFile(filePath, content, 0644)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write precompiled file %q: %w", filePath, err)
+		}
 	}
 
-	return filepath.Join(pipelineDir, newPipelineFileName), nil
+	return processedPipeline, nil
 }
 
-func processPipelineForEV2(p *pipeline.Pipeline, vars config.Variables) error {
-	_, scopeBoundVars := EV2Mapping(vars, []string{})
+func readReferencedPipelineFiles(p *pipeline.Pipeline) (map[string][]byte, error) {
+	// switch to pipeline directory to ensure relative paths are resolvable
+	_, restoreDir, err := p.EnterPipelineDir()
+	if err != nil {
+		return nil, fmt.Errorf("failed to enter pipeline directory: %w", err)
+	}
+	defer restoreDir()
+
+	referencedFiles := make(map[string][]byte)
 	for _, rg := range p.ResourceGroups {
 		for _, step := range rg.Steps {
 			if step.Parameters != "" {
-				newParameterFilePath, err := precompileFileAndStore(step.Parameters, scopeBoundVars)
+				paramFileContent, err := os.ReadFile(step.Parameters)
 				if err != nil {
-					return err
+					return nil, fmt.Errorf("failed to read parameter file %q: %w", step.Parameters, err)
 				}
-				step.Parameters = newParameterFilePath
+				referencedFiles[step.Parameters] = paramFileContent
 			}
 		}
 	}
-	return nil
+	return referencedFiles, nil
 }
 
-func precompileFileAndStore(filePath string, vars map[string]interface{}) (string, error) {
-	preprocessedBytes, err := config.PreprocessFile(filePath, vars)
+func processPipelineForEV2(p *pipeline.Pipeline, referencedFiles map[string][]byte, vars config.Variables) (*pipeline.Pipeline, map[string][]byte, error) {
+	processingPipeline, err := p.DeepCopy(buildPrefixedFilePath(p.PipelineFilePath(), precompiledPrefix))
 	if err != nil {
-		return "", err
+		return nil, nil, err
 	}
-	newFilePath := buildPrefixedFilePath(filePath, "ev2-precompiled-")
-	err = os.WriteFile(newFilePath, preprocessedBytes, 0644)
-	if err != nil {
-		return "", err
+	processedFiles := make(map[string][]byte)
+	_, scopeBoundVars := EV2Mapping(vars, []string{})
+	for _, rg := range processingPipeline.ResourceGroups {
+		for _, step := range rg.Steps {
+			// preprocess the parameters file with scopebinding variables
+			if step.Parameters != "" {
+				paramFileContent, ok := referencedFiles[step.Parameters]
+				if !ok {
+					return nil, nil, fmt.Errorf("parameter file %q not found", step.Parameters)
+				}
+				preprocessedBytes, err := config.PreprocessContent(paramFileContent, scopeBoundVars)
+				if err != nil {
+					return nil, nil, err
+				}
+				newParameterFilePath := buildPrefixedFilePath(step.Parameters, precompiledPrefix)
+				processedFiles[newParameterFilePath] = preprocessedBytes
+				step.Parameters = newParameterFilePath
+			}
+		}
 	}
-	return newFilePath, nil
+	return processingPipeline, processedFiles, nil
 }
 
 func buildPrefixedFilePath(path, prefix string) string {

tooling/templatize/pkg/ev2/pipeline_test.go

@@ -1,38 +1,42 @@
 package ev2
 
 import (
-	"fmt"
-	"os"
 	"testing"
 
+	"gopkg.in/yaml.v3"
+
+	"github.com/Azure/ARO-HCP/tooling/templatize/internal/testutil"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
 
-func TestPrecompilePipelineForEV2(t *testing.T) {
-	defer func() {
-		_ = os.Remove("../../testdata/ev2-precompiled-pipeline.yaml")
-		_ = os.Remove("../../testdata/ev2-precompiled-test.bicepparam")
-	}()
-
+func TestProcessPipelineForEV2(t *testing.T) {
 	configProvider := config.NewConfigProvider("../../testdata/config.yaml")
-	vars, err := configProvider.GetVariables("public", "int", "", newEv2ConfigReplacements())
+	vars, err := configProvider.GetVariables("public", "int", "", NewEv2ConfigReplacements())
 	if err != nil {
 		t.Errorf("failed to get variables: %v", err)
 	}
-	newPipelinePath, err := PrecompilePipelineForEV2("../../testdata/pipeline.yaml", vars)
+	originalPipeline, err := pipeline.NewPipelineFromFile("../../testdata/pipeline.yaml", vars)
+	if err != nil {
+		t.Errorf("failed to read new pipeline: %v", err)
+	}
+	files := make(map[string][]byte)
+	files["test.bicepparam"] = []byte("param regionRG = '{{ .regionRG }}'")
+
+	newPipeline, newFiles, err := processPipelineForEV2(originalPipeline, files, vars)
 	if err != nil {
 		t.Errorf("failed to precompile pipeline: %v", err)
 	}
 
-	p, err := pipeline.NewPipelineFromFile(newPipelinePath, vars)
+	// verify pipeline
+	pipelineContent, err := yaml.Marshal(newPipeline)
 	if err != nil {
-		t.Errorf("failed to read new pipeline: %v", err)
+		t.Errorf("failed to marshal processed pipeline: %v", err)
 	}
-	fmt.Println(p)
-	expectedParamsPath := "ev2-precompiled-test.bicepparam"
-	if p.ResourceGroups[0].Steps[1].Parameters != expectedParamsPath {
-		t.Errorf("expected parameters path %v, but got %v", expectedParamsPath, p.ResourceGroups[0].Steps[1].Parameters)
+	testutil.CompareWithFixture(t, pipelineContent, testutil.WithExtension("pipeline.yaml"))
+
+	// verify referenced files
+	for filePath, content := range newFiles {
+		testutil.CompareWithFixture(t, content, testutil.WithExtension(filePath))
 	}
-	// TODO improve test, check against fixture
 }

tooling/templatize/pkg/ev2/utils.go

@@ -10,7 +10,7 @@ import (
 // This package contains helper functions to extract EV2 conformant data from a config.yaml file.
 //
 
-func newEv2ConfigReplacements() *config.ConfigReplacements {
+func NewEv2ConfigReplacements() *config.ConfigReplacements {
 	return config.NewConfigReplacements(
 		"$location()",
 		"$(regionShortName)",
@@ -23,7 +23,7 @@ func newEv2ConfigReplacements() *config.ConfigReplacements {
 // The variable values are formatted to contain EV2 $location(), $stamp() and $(serviceConfigVar) variables.
 // This function is useful to get the variables to fill the `Settings` section of an EV2 `ServiceConfig.json“
 func GetNonRegionalServiceConfigVariables(configProvider config.ConfigProvider, cloud, deployEnv string) (config.Variables, error) {
-	return configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements())
+	return configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
 }
 
 // GetRegionalServiceConfigVariableOverrides returns the regional overrides of a config.yaml file.
@@ -36,7 +36,7 @@ func GetRegionalServiceConfigVariableOverrides(configProvider config.ConfigProvi
 	}
 	overrides := make(map[string]config.Variables)
 	for _, region := range regions {
-		regionOverrides, err := configProvider.GetRegionOverrides(cloud, deployEnv, region, newEv2ConfigReplacements())
+		regionOverrides, err := configProvider.GetRegionOverrides(cloud, deployEnv, region, NewEv2ConfigReplacements())
 		if err != nil {
 			return nil, err
 		}
@@ -49,7 +49,7 @@ func GetRegionalServiceConfigVariableOverrides(configProvider config.ConfigProvi
 // It uses the provided configProvider to fetch the variables, flattens them into a __VAR__ = $config(var) formatted map.
 // This function is useful to get the find/replace pairs for an EV2 `ScopeBinding.json`
 func ScopeBindingVariables(configProvider config.ConfigProvider, cloud, deployEnv string) (map[string]string, error) {
-	vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements())
+	vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +65,7 @@ func ScopeBindingVariables(configProvider config.ConfigProvider, cloud, deployEn
 // while maintaining EV2 conformant system variables.
 // This function is useful to process a pipeline.yaml file so that it contains EV2 system variables.
 func PreprocessFileForEV2SystemVars(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) {
-	vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements())
+	vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
 	if err != nil {
 		return nil, err
 	}
@@ -77,7 +77,7 @@ func PreprocessFileForEV2SystemVars(configProvider config.ConfigProvider, cloud,
 // This function is useful to process bicepparam files so that they can be used within EV2 together
 // with scopebinding.
 func PreprocessFileForEV2ScopeBinding(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) {
-	vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements())
+	vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements())
 	if err != nil {
 		return nil, err
 	}

tooling/templatize/pkg/pipeline/arm.go

@@ -10,7 +10,7 @@ import (
 	"github.com/go-logr/logr"
 )
 
-func (s *step) runArmStep(ctx context.Context, executionTarget *ExecutionTarget, options *PipelineRunOptions) error {
+func (s *Step) runArmStep(ctx context.Context, executionTarget *ExecutionTarget, options *PipelineRunOptions) error {
 	logger := logr.FromContextOrDiscard(ctx)
 
 	// Transform Bicep to ARM
@@ -59,7 +59,7 @@ func (s *step) runArmStep(ctx context.Context, executionTarget *ExecutionTarget,
 	return nil
 }
 
-func (s *step) ensureResourceGroupExists(ctx context.Context, executionTarget *ExecutionTarget) error {
+func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget *ExecutionTarget) error {
 	// Create a new Azure identity client
 	cred, err := azidentity.NewDefaultAzureCredential(nil)
 	if err != nil {

tooling/templatize/pkg/pipeline/common.go

@@ -0,0 +1,47 @@
+package pipeline
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v3"
+)
+
+func (p *Pipeline) DeepCopy(newPipelineFilePath string) (*Pipeline, error) {
+	copy := new(Pipeline)
+	data, err := yaml.Marshal(p)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal pipeline: %v", err)
+	}
+	err = yaml.Unmarshal(data, copy)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal pipeline: %v", err)
+	}
+	copy.pipelineFilePath = newPipelineFilePath
+	return copy, nil
+}
+
+func (p *Pipeline) PipelineFilePath() string {
+	return p.pipelineFilePath
+}
+
+func (p *Pipeline) EnterPipelineDir() (string, func(), error) {
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return "", nil, err
+	}
+
+	pipelineDir, err := filepath.Abs(filepath.Dir(p.pipelineFilePath))
+	if err != nil {
+		return "", nil, err
+	}
+	err = os.Chdir(pipelineDir)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return pipelineDir, func() {
+		_ = os.Chdir(currentDir)
+	}, nil
+}

tooling/templatize/pkg/pipeline/inspect.go

@@ -8,7 +8,7 @@ import (
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 )
 
-type StepInspectScope func(*step, *PipelineInspectOptions, io.Writer) error
+type StepInspectScope func(*Step, *PipelineInspectOptions, io.Writer) error
 
 func NewStepInspectScopes() map[string]StepInspectScope {
 	return map[string]StepInspectScope{
@@ -44,7 +44,7 @@ func (p *Pipeline) Inspect(ctx context.Context, options *PipelineInspectOptions,
 	return fmt.Errorf("step %q not found", options.Step)
 }
 
-func inspectVars(s *step, options *PipelineInspectOptions, writer io.Writer) error {
+func inspectVars(s *Step, options *PipelineInspectOptions, writer io.Writer) error {
 	var envVars map[string]string
 	var err error
 	switch s.Action {