From 0389cde3cea64cb6de557b3d9e973b94479e2cef Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Thu, 3 Aug 2023 14:16:36 -0700 Subject: [PATCH 01/15] implement terraform support --- go.mod | 19 +- pkg/cmd/crd.go | 53 ++-- .../valid/input-require/variables.tf | 43 +++ pkg/cmd/fakes/terraform/valid/input/main.tf | 148 ++++++++++ .../fakes/terraform/valid/input/variables.tf | 44 +++ .../valid/output/full-template-require.yaml | 62 +++++ .../terraform/valid/output/full-template.yaml | 64 +++++ .../terraform/valid/output/properties.yaml | 40 +++ pkg/cmd/template.go | 1 - pkg/cmd/terraform.go | 257 ++++++++++++++++++ pkg/cmd/terraform_test.go | 103 +++++++ pkg/cmd/utils.go | 100 +++++++ pkg/models/structs.go | 16 ++ 13 files changed, 908 insertions(+), 42 deletions(-) create mode 100644 pkg/cmd/fakes/terraform/valid/input-require/variables.tf create mode 100644 pkg/cmd/fakes/terraform/valid/input/main.tf create mode 100644 pkg/cmd/fakes/terraform/valid/input/variables.tf create mode 100644 pkg/cmd/fakes/terraform/valid/output/full-template-require.yaml create mode 100644 pkg/cmd/fakes/terraform/valid/output/full-template.yaml create mode 100644 pkg/cmd/fakes/terraform/valid/output/properties.yaml create mode 100644 pkg/cmd/terraform.go create mode 100644 pkg/cmd/terraform_test.go create mode 100644 pkg/cmd/utils.go diff --git a/go.mod b/go.mod index 06a6dac..3e396dd 100644 --- a/go.mod +++ b/go.mod @@ -5,17 +5,22 @@ go 1.19 require ( github.com/fatih/color v1.15.0 github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/terraform-config-inspect v0.0.0-20230614215431-f32df32a01cd + github.com/itchyny/gojq v0.12.13 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.2 github.com/onsi/ginkgo/v2 v2.9.7 github.com/onsi/gomega v1.27.8 github.com/spf13/cobra v1.7.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.27.3 - k8s.io/apimachinery v0.27.3 - k8s.io/client-go v0.27.3 + k8s.io/api v0.27.4 + k8s.io/apimachinery v0.27.4 + k8s.io/client-go v0.27.4 + sigs.k8s.io/yaml v1.3.0 ) require ( + github.com/agext/levenshtein v1.2.2 // indirect + github.com/apparentlymart/go-textseg v1.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/go-logr/logr v1.2.4 // indirect @@ -31,17 +36,22 @@ require ( github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f // indirect + github.com/hashicorp/hcl/v2 v2.0.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/zclconf/go-cty v1.1.0 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.11.0 // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect @@ -59,5 +69,4 @@ require ( k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index b89bde4..63ed8f9 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -4,14 +4,13 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "strings" "github.com/cnoe-io/cnoe-cli/pkg/models" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -116,7 +115,7 @@ func defs(dir string, depth int) []string { var out []string base, _ := filepath.Abs(dir) - files, _ := ioutil.ReadDir(base) + files, _ := os.ReadDir(base) for _, file := range files { f := filepath.Join(base, file.Name()) stat, _ := os.Stat(f) @@ -151,7 +150,7 @@ func writeSchema(stdout, stderr io.Writer, outputDir string, defs []string) (cmd } for _, def := range defs { - data, err := ioutil.ReadFile(def) + data, err := os.ReadFile(def) if err != nil { continue } @@ -159,6 +158,7 @@ func writeSchema(stdout, stderr io.Writer, outputDir string, defs []string) (cmd var doc models.Definition err = yaml.Unmarshal(data, &doc) if err != nil { + fmt.Printf("failed to read %s. This file will be excluded. %s", def, err) continue } @@ -182,8 +182,7 @@ func writeSchema(stdout, stderr io.Writer, outputDir string, defs []string) (cmd } else { value, err = ConvertMap(v) if err != nil { - fmt.Fprintf(stdout, "failed %s: %s \n", def, err.Error()) - continue + return cmdOutput{}, err } } @@ -243,7 +242,7 @@ func writeSchema(stdout, stderr io.Writer, outputDir string, defs []string) (cmd } template := fmt.Sprintf("%s/%s.yaml", templateOutputDir, strings.ToLower(resourceName)) - err = ioutil.WriteFile(template, []byte(wrapperData), 0644) + err = os.WriteFile(template, []byte(wrapperData), 0644) if err != nil { fmt.Fprintf(stdout, "failed %s: %s \n", def, err.Error()) continue @@ -261,7 +260,7 @@ func writeToTemplate( templateFile string, outputPath string, identifiedResources []string, position int, templateName, templateTitle, templateDescription string, ) error { - templateData, err := ioutil.ReadFile(templateFile) + templateData, err := os.ReadFile(templateFile) if err != nil { return err } @@ -316,7 +315,7 @@ func writeToTemplate( return err } - err = ioutil.WriteFile(fmt.Sprintf("%s/template.yaml", outputPath), outputData, 0644) + err = os.WriteFile(fmt.Sprintf("%s/template.yaml", outputPath), outputData, 0644) if err != nil { return err } @@ -334,32 +333,26 @@ func ConvertSlice(strSlice []string) []interface{} { } func ConvertMap(originalData interface{}) (map[string]interface{}, error) { - originalMap, ok := originalData.(map[interface{}]interface{}) + originalMap, ok := originalData.(map[string]interface{}) if !ok { - return nil, errors.New("failed to convert to interface map") + return nil, errors.New("conversion failed: data is not map[string]interface{}") } convertedMap := make(map[string]interface{}) for key, value := range originalMap { - strKey, ok := key.(string) - if !ok { - // Skip the key if it cannot be converted to string - continue - } - switch v := value.(type) { case map[interface{}]interface{}: // If the value is a nested map, recursively convert it var err error - convertedMap[strKey], err = ConvertMap(v) + convertedMap[key], err = ConvertMap(v) if err != nil { - return nil, errors.New(fmt.Sprintf("failed to convert for key %s", strKey)) + return nil, errors.New(fmt.Sprintf("failed to convert for key %s", key)) } case int: - convertedMap[strKey] = int64(v) + convertedMap[key] = int64(v) case int32: - convertedMap[strKey] = int64(v) + convertedMap[key] = int64(v) case []interface{}: dv := make([]interface{}, len(v)) for i, ve := range v { @@ -367,7 +360,7 @@ func ConvertMap(originalData interface{}) (map[string]interface{}, error) { case map[interface{}]interface{}: ivec, err := ConvertMap(ive) if err != nil { - return nil, errors.New(fmt.Sprintf("failed to convert for key %s", strKey)) + return nil, errors.New(fmt.Sprintf("failed to convert for key %s", key)) } dv[i] = ivec case int: @@ -378,10 +371,10 @@ func ConvertMap(originalData interface{}) (map[string]interface{}, error) { dv[i] = ive } } - convertedMap[strKey] = dv + convertedMap[key] = dv default: // Otherwise, add the key-value pair to the converted map - convertedMap[strKey] = v + convertedMap[key] = v } } @@ -395,15 +388,3 @@ func isXRD(m models.Definition) bool { func isCRD(m models.Definition) bool { return m.Kind == KindCRD } - -func isDirectory(path string) bool { - // Get file information - info, err := os.Stat(path) - if err != nil { - // Error occurred, path does not exist or cannot be accessed - return false - } - - // Check if the path is a directory - return info.Mode().IsDir() -} diff --git a/pkg/cmd/fakes/terraform/valid/input-require/variables.tf b/pkg/cmd/fakes/terraform/valid/input-require/variables.tf new file mode 100644 index 0000000..a539975 --- /dev/null +++ b/pkg/cmd/fakes/terraform/valid/input-require/variables.tf @@ -0,0 +1,43 @@ +variable "name" { + description = "Name of the VPC and EKS Cluster" + type = string + default = "emr-eks-ack" +} + +variable "region" { + description = "Region" + type = string +} + +variable "eks_cluster_version" { + description = "EKS Cluster version" + type = string + default = "1.27" +} + +variable "tags" { + description = "Default tags" + type = map( string ) + default = { + "env" = "test" + } +} + +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.1.0.0/16" +} + +# Only two Subnets for with low IP range for internet access +variable "public_subnets" { + description = "Public Subnets CIDRs. 62 IPs per Subnet" + type = list(string) + default = ["10.1.255.128/26", "10.1.255.192/26"] +} + +variable "private_subnets" { + description = "Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet" + type = list(string) + default = ["10.1.0.0/17", "10.1.128.0/18"] +} diff --git a/pkg/cmd/fakes/terraform/valid/input/main.tf b/pkg/cmd/fakes/terraform/valid/input/main.tf new file mode 100644 index 0000000..a5296b5 --- /dev/null +++ b/pkg/cmd/fakes/terraform/valid/input/main.tf @@ -0,0 +1,148 @@ +provider "aws" { + region = local.region +} + +# ECR always authenticates with `us-east-1` region +# Docs -> https://docs.aws.amazon.com/AmazonECR/latest/public/public-registries.html +provider "aws" { + alias = "ecr" + region = "us-east-1" +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token + } +} + +data "aws_eks_cluster_auth" "this" { + name = module.eks.cluster_name +} + +data "aws_ecrpublic_authorization_token" "token" { + provider = aws.ecr +} + +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + +locals { + name = var.name + region = var.region + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 2) + + tags = merge(var.tags, { + Blueprint = local.name + GithubRepo = "github.com/awslabs/data-on-eks" + }) +} + +#--------------------------------------------------------------- +# EKS Cluster +#--------------------------------------------------------------- + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.15" + + cluster_name = local.name + cluster_version = var.eks_cluster_version + + cluster_endpoint_private_access = true # if true, Kubernetes API requests within your cluster's VPC (such as node to control plane communication) use the private VPC endpoint + cluster_endpoint_public_access = true # if true, Your cluster API server is accessible from the internet. You can, optionally, limit the CIDR blocks that can access the public endpoint. + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + manage_aws_auth_configmap = true + aws_auth_roles = [ + { + # Required for EMR on EKS virtual cluster + rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/AWSServiceRoleForAmazonEMRContainers" + username = "emr-containers" + }, + ] + + #--------------------------------------- + # Note: This can further restricted to specific required for each Add-on and your application + #--------------------------------------- + # Extend cluster security group rules + cluster_security_group_additional_rules = { + ingress_nodes_ephemeral_ports_tcp = { + description = "Nodes on ephemeral ports" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "ingress" + source_node_security_group = true + } + } + + # Extend node-to-node security group rules + node_security_group_additional_rules = { + # Extend node-to-node security group rules. Recommended and required for the Add-ons + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + } + + eks_managed_node_group_defaults = { + iam_role_additional_policies = { + # Not required, but used in the example to access the nodes to inspect mounted volumes + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + } + eks_managed_node_groups = { + # We recommend to have a MNG to place your critical workloads and add-ons + # Then rely on Karpenter to scale your workloads + # You can also make uses on nodeSelector and Taints/tolerations to spread workloads on MNG or Karpenter provisioners + core_node_group = { + name = "core-node-group" + description = "EKS managed node group example launch template" + + min_size = 1 + max_size = 9 + desired_size = 3 + + instance_types = ["m5.xlarge"] + + ebs_optimized = true + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 100 + volume_type = "gp3" + } + } + } + + labels = { + WorkerType = "ON_DEMAND" + NodeGroupType = "core" + } + + tags = { + Name = "core-node-grp", + "karpenter.sh/discovery" = local.name + } + } + } + + tags = local.tags +} diff --git a/pkg/cmd/fakes/terraform/valid/input/variables.tf b/pkg/cmd/fakes/terraform/valid/input/variables.tf new file mode 100644 index 0000000..d7e42be --- /dev/null +++ b/pkg/cmd/fakes/terraform/valid/input/variables.tf @@ -0,0 +1,44 @@ +variable "name" { + description = "Name of the VPC and EKS Cluster" + type = string + default = "emr-eks-ack" +} + +variable "region" { + description = "Region" + type = string + default = "us-west-2" +} + +variable "eks_cluster_version" { + description = "EKS Cluster version" + type = string + default = "1.27" +} + +variable "tags" { + description = "Default tags" + type = map( string ) + default = { + "env" = "test" + } +} + +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.1.0.0/16" +} + +# Only two Subnets for with low IP range for internet access +variable "public_subnets" { + description = "Public Subnets CIDRs. 62 IPs per Subnet" + type = list(string) + default = ["10.1.255.128/26", "10.1.255.192/26"] +} + +variable "private_subnets" { + description = "Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet" + type = list(string) + default = ["10.1.0.0/17", "10.1.128.0/18"] +} diff --git a/pkg/cmd/fakes/terraform/valid/output/full-template-require.yaml b/pkg/cmd/fakes/terraform/valid/output/full-template-require.yaml new file mode 100644 index 0000000..8a4c2db --- /dev/null +++ b/pkg/cmd/fakes/terraform/valid/output/full-template-require.yaml @@ -0,0 +1,62 @@ +apiVersion: scaffolder.backstage.io/v1beta3 +kind: Template +metadata: + name: deploy-resources + title: Deploy Resources + description: Deploy Resource to Kubernetes +spec: + owner: guest + type: service + parameters: + - title: Choose Resource + description: Select a AWS resource to add to your repository. + properties: + path: + type: string + description: path to place this file into + default: kustomize/base + eks_cluster_version: + type: string + description: EKS Cluster version + default: "1.27" + name: + type: string + description: Name of the VPC and EKS Cluster + default: emr-eks-ack + private_subnets: + type: array + description: Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet + default: + - 10.1.0.0/17 + - 10.1.128.0/18 + items: + type: string + public_subnets: + type: array + description: Public Subnets CIDRs. 62 IPs per Subnet + default: + - 10.1.255.128/26 + - 10.1.255.192/26 + items: + type: string + region: + type: string + description: Region + tags: + title: tags + type: object + description: Default tags + additionalProperties: + type: string + vpc_cidr: + type: string + description: VPC CIDR + default: 10.1.0.0/16 + required: + - awsResources + - name + - region + steps: + - id: verify + name: verify + action: cnoe:verify:dependency diff --git a/pkg/cmd/fakes/terraform/valid/output/full-template.yaml b/pkg/cmd/fakes/terraform/valid/output/full-template.yaml new file mode 100644 index 0000000..83c53e5 --- /dev/null +++ b/pkg/cmd/fakes/terraform/valid/output/full-template.yaml @@ -0,0 +1,64 @@ +# This template uses $yaml special keys to include objects from different files. For this to work, the catalog type must be "url". Specifically, it must be http, e.g. Using something like file://abc/def/template-add-aws-resources.yaml does not work. +apiVersion: scaffolder.backstage.io/v1beta3 +kind: Template +metadata: + name: deploy-resources + title: Deploy Resources + description: Deploy Resource to Kubernetes +spec: + owner: guest + type: service + # these are the steps which are rendered in the frontend with the form input + parameters: + - title: Choose Resource + description: Select a AWS resource to add to your repository. + properties: + path: + type: string + description: path to place this file into + default: kustomize/base + eks_cluster_version: + type: string + description: EKS Cluster version + default: "1.27" + name: + type: string + description: Name of the VPC and EKS Cluster + default: emr-eks-ack + private_subnets: + type: array + description: Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet + default: + - 10.1.0.0/17 + - 10.1.128.0/18 + items: + type: string + public_subnets: + type: array + description: Public Subnets CIDRs. 62 IPs per Subnet + default: + - 10.1.255.128/26 + - 10.1.255.192/26 + items: + type: string + region: + type: string + description: Region + default: us-west-2 + tags: + title: tags + type: object + description: Default tags + additionalProperties: + type: string + vpc_cidr: + type: string + description: VPC CIDR + default: 10.1.0.0/16 + required: + - awsResources + - name + steps: + - id: verify + name: verify + action: cnoe:verify:dependency diff --git a/pkg/cmd/fakes/terraform/valid/output/properties.yaml b/pkg/cmd/fakes/terraform/valid/output/properties.yaml new file mode 100644 index 0000000..45354fe --- /dev/null +++ b/pkg/cmd/fakes/terraform/valid/output/properties.yaml @@ -0,0 +1,40 @@ +properties: + eks_cluster_version: + type: string + description: EKS Cluster version + default: "1.27" + name: + type: string + description: Name of the VPC and EKS Cluster + default: emr-eks-ack + private_subnets: + type: array + description: Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet + default: + - 10.1.0.0/17 + - 10.1.128.0/18 + items: + type: string + public_subnets: + type: array + description: Public Subnets CIDRs. 62 IPs per Subnet + default: + - 10.1.255.128/26 + - 10.1.255.192/26 + items: + type: string + region: + type: string + description: Region + default: us-west-2 + tags: + title: tags + type: object + description: Default tags + additionalProperties: + type: string + vpc_cidr: + type: string + description: VPC CIDR + default: 10.1.0.0/16 +required: [] diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index dfd76e4..a028fae 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -11,7 +11,6 @@ var templateCmd = &cobra.Command{ func init() { rootCmd.AddCommand(templateCmd) - templateCmd.PersistentFlags().StringVarP(&inputDir, "inputDir", "i", "", "input directory for CRDs and XRDs to be templatized") templateCmd.PersistentFlags().StringVarP(&outputDir, "outputDir", "o", "", "output directory for backstage templates to be stored in") diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go new file mode 100644 index 0000000..8172876 --- /dev/null +++ b/pkg/cmd/terraform.go @@ -0,0 +1,257 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/cnoe-io/cnoe-cli/pkg/models" + "github.com/hashicorp/terraform-config-inspect/tfconfig" + "github.com/spf13/cobra" +) + +var ( + tfCmd = &cobra.Command{ + Use: "tf", + Short: "Generate backstage templates from Terraform variables", + PreRunE: func(cmd *cobra.Command, args []string) error { + if !isDirectory(inputDir) { + return errors.New("inputDir and ouputDir entries need to be directories") + } + return nil + }, + RunE: tfE, + } + depth uint32 + insertionPoint string +) + +func init() { + tfCmd.Flags().Uint32Var(&depth, "depth", 2, "depth from given directory to search for TF modules") + tfCmd.Flags().StringVarP(&templatePath, "templatePath", "t", "scaffolding/template.yaml", "path to the template to be augmented with backstage info") + tfCmd.Flags().StringVarP(&insertionPoint, "insertAt", "p", "", "jq path within the template to insert backstage info") + crdCmd.AddCommand(tfCmd) +} + +func tfE(cmd *cobra.Command, args []string) error { + return terraform(cmd.Context(), inputDir, outputDir, templatePath, insertionPoint) +} + +func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string) error { + mods := getModules(inputDir, 0) + if len(mods) == 0 { + return fmt.Errorf("could not find any TF modules in given directorr: %s", inputDir) + } + + for i := range mods { + path := mods[i] + mod, diag := tfconfig.LoadModule(path) + if diag.HasErrors() { + return diag.Err() + } + if len(mod.Variables) == 0 { + fmt.Println(fmt.Sprintf("module %s does not have variables", path)) + continue + } + params := make(map[string]models.BackstageParamFields) + required := make([]string, 0) + for j := range mod.Variables { + params[j] = convertVariable(*mod.Variables[j]) + if mod.Variables[j].Required { + required = append(required, j) + } + } + filePath := filepath.Join(outputDir, fmt.Sprintf("%s.yaml", filepath.Base(inputDir))) + err := handleOutput(ctx, filePath, templatePath, insertionPoint, params, required) + if err != nil { + log.Println(err) + } + } + return nil +} + +func handleOutput(ctx context.Context, outputFile, templatePath, insertionPoint string, properties map[string]models.BackstageParamFields, required []string) error { + if templatePath != "" && insertionPoint != "" { + input := insertAtInput{ + templatePath: templatePath, + jqPathExpression: insertionPoint, + properties: properties, + required: required, + } + t, err := insertAt(ctx, input) + if err != nil { + return fmt.Errorf("failed to insert to given template: %s", err) + } + return writeOutput(t, outputFile) + } + t := map[string]any{ + "properties": properties, + "required": required, + } + return writeOutput(t, outputFile) +} + +func getModules(inputDir string, currentDepth uint32) []string { + if currentDepth > depth { + return nil + } + if tfconfig.IsModuleDir(inputDir) { + return []string{inputDir} + } + base, _ := filepath.Abs(inputDir) + files, _ := os.ReadDir(base) + out := make([]string, 1) + for _, file := range files { + f := filepath.Join(base, file.Name()) + mods := getModules(f, currentDepth+1) + out = append(out, mods...) + } + return out +} + +func convertVariable(tfVar tfconfig.Variable) models.BackstageParamFields { + tfType := cleanString(tfVar.Type) + t := mapType(tfType) + if isPrimitive(tfType) { + b := models.BackstageParamFields{ + Type: t, + } + if tfVar.Description != "" { + b.Description = tfVar.Description + } + if tfVar.Default != nil { + b.Default = tfVar.Default + } + return b + } + + if t == "array" { + return convertArray(tfVar) + } + if t == "object" { + return convertObject(tfVar) + } + return models.BackstageParamFields{} +} + +func convertArray(tfVar tfconfig.Variable) models.BackstageParamFields { + tfType := cleanString(tfVar.Type) + nestedType := getNestedType(tfType) + nestedTfVar := tfconfig.Variable{ + Name: fmt.Sprintf("%s-a", tfVar.Name), + Type: nestedType, + } + nestedItems := convertVariable(nestedTfVar) + out := models.BackstageParamFields{ + Type: "array", + Description: tfVar.Description, + Default: tfVar.Default, + Items: &nestedItems, + } + if strings.HasPrefix(tfType, "set") { + u := true + out.UniqueItems = &u + } + return out +} + +func convertObjectDefaults(tfVar tfconfig.Variable) map[string]*models.BackstageParamFields { + // build default values by taking default's key and type. Must be done for primitives only. + properties := make(map[string]*models.BackstageParamFields) + nestedType := getNestedType(cleanString(tfVar.Type)) + if tfVar.Default != nil { + d, ok := tfVar.Default.(map[string]any) + if !ok { + log.Fatalf("could not determine default type of %s\n", tfVar.Default) + } + for k := range d { + defaultProp := convertVariable(tfconfig.Variable{ + Name: k, + Type: nestedType, + Default: d[k], + }) + properties[k] = &defaultProp + } + } + return properties +} + +func convertObject(tfVar tfconfig.Variable) models.BackstageParamFields { + out := models.BackstageParamFields{ + Title: tfVar.Name, + Type: mapType(cleanString(tfVar.Type)), + Description: tfVar.Description, + } + + nestedType := getNestedType(cleanString(tfVar.Type)) + if isPrimitive(nestedType) { + p := models.AdditionalProperties{Type: mapType(nestedType)} + out.AdditionalProperties = &p + // defaults for object type is broken in Backstage atm. In the UI, the default values cannot be removed. + //properties := convertObjectDefaults(tfVar) + //if len(properties) > 0 { + // out.Properties = properties + //} + } else { + name := fmt.Sprintf("%s-n", tfVar.Name) + nestedTfVar := tfconfig.Variable{ + Name: name, + Type: nestedType, + } + converted := convertVariable(nestedTfVar) + out.Properties = map[string]*models.BackstageParamFields{ + name: &converted, + } + } + return out +} + +func cleanString(input string) string { + return strings.ReplaceAll(input, " ", "") +} + +func isPrimitive(s string) bool { + return s == "string" || s == "number" || s == "bool" +} + +func isNestedPrimitive(s string) bool { + nested := strings.HasPrefix(s, "object(") || strings.HasPrefix(s, "map(") || strings.HasPrefix(s, "list(") + if nested { + return isPrimitive(getNestedType(s)) + } + return false +} + +func getNestedType(s string) string { + if strings.HasPrefix(s, "object(") { + return strings.TrimSuffix(strings.SplitAfterN(s, "object(", 1)[1], ")") + } + if strings.HasPrefix(s, "map(") { + return strings.TrimSuffix(strings.SplitAfterN(s, "map(", 2)[1], ")") + } + if strings.HasPrefix(s, "list(") { + return strings.TrimSuffix(strings.SplitAfterN(s, "list(", 2)[1], ")") + } + return s +} + +func mapType(tfType string) string { + switch { + case tfType == "string": + return "string" + case tfType == "number": + return "number" + case tfType == "bool": + return "boolean" + case strings.HasPrefix(tfType, "object"), strings.HasPrefix(tfType, "map"): + return "object" + case strings.HasPrefix(tfType, "list"), strings.HasPrefix(tfType, "set"): + return "array" + default: + return "string" + } +} diff --git a/pkg/cmd/terraform_test.go b/pkg/cmd/terraform_test.go new file mode 100644 index 0000000..12c740c --- /dev/null +++ b/pkg/cmd/terraform_test.go @@ -0,0 +1,103 @@ +package cmd + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "os" + "path/filepath" +) + +var _ = Describe("Terraform Template", func() { + var ( + tempDir string + outputDir string + ) + + const ( + inputDir = "./fakes/terraform/valid/input" + inputDirWithRequire = "./fakes/terraform/valid/input-require" + expectedPropertyFile = "./fakes/terraform/valid/output/properties.yaml" + expectedTemplateFile = "./fakes/terraform/valid/output/full-template.yaml" + expectedTemplateFileWithRequire = "./fakes/terraform/valid/output/full-template-require.yaml" + targetTemplateFile = "./fakes/template/input-template.yaml" + ) + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "test-temp") + Expect(err).NotTo(HaveOccurred()) + + outputDir = filepath.Join(tempDir, "output") + err = os.Mkdir(outputDir, 0755) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := os.RemoveAll(tempDir) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("with valid input and no target template specified", func() { + BeforeEach(func() { + err := terraform(context.Background(), inputDir, outputDir, "", "") + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create the skeleton file for valid definitions", func() { + generatedData, err := os.ReadFile(fmt.Sprintf("%s/input.yaml", outputDir)) + Expect(err).NotTo(HaveOccurred()) + expectedData, err := os.ReadFile(expectedPropertyFile) + Expect(err).NotTo(HaveOccurred()) + Expect(generatedData).To(MatchYAML(expectedData)) + }) + + }) + Context("with valid input and a target template specified", func() { + BeforeEach(func() { + err := terraform(context.Background(), inputDir, outputDir, targetTemplateFile, ".spec.parameters[0]") + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create the template file with properties merged", func() { + generatedData, err := os.ReadFile(fmt.Sprintf("%s/input.yaml", outputDir)) + Expect(err).NotTo(HaveOccurred()) + + expectedData, err := os.ReadFile(expectedTemplateFile) + + var generated map[string]any + err = yaml.Unmarshal(generatedData, &generated) + var expected map[string]any + err = yaml.Unmarshal(expectedData, &expected) + Expect(err).NotTo(HaveOccurred()) + Expect(generated).To(Equal(expected)) + }) + + It("should create the template file with properties merged and requirements updated", func() { + generatedData, err := os.ReadFile(fmt.Sprintf("%s/input.yaml", outputDir)) + Expect(err).NotTo(HaveOccurred()) + expectedData, err := os.ReadFile(expectedTemplateFile) + Expect(err).NotTo(HaveOccurred()) + Expect(generatedData).To(MatchYAML(expectedData)) + }) + + }) + Context("with valid input with required variable and a target template specified", func() { + BeforeEach(func() { + err := terraform(context.Background(), inputDirWithRequire, outputDir, targetTemplateFile, ".spec.parameters[0]") + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create the template file with properties merged and requirements updated", func() { + generatedData, err := os.ReadFile(fmt.Sprintf("%s/input-require.yaml", outputDir)) + Expect(err).NotTo(HaveOccurred()) + expectedData, err := os.ReadFile(expectedTemplateFileWithRequire) + Expect(err).NotTo(HaveOccurred()) + Expect(generatedData).To(MatchYAML(expectedData)) + }) + }) +}) diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go new file mode 100644 index 0000000..06f12ca --- /dev/null +++ b/pkg/cmd/utils.go @@ -0,0 +1,100 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + "strings" + + "github.com/cnoe-io/cnoe-cli/pkg/models" + "github.com/itchyny/gojq" + yamlv3 "gopkg.in/yaml.v3" + "sigs.k8s.io/yaml" +) + +type NotFoundError struct { + Err error +} + +func (n NotFoundError) Error() string { + return n.Error() +} + +type insertAtInput struct { + templatePath string + jqPathExpression string + properties map[string]models.BackstageParamFields + required []string +} + +func isDirectory(path string) bool { + // Get file information + info, err := os.Stat(path) + if err != nil { + // Error occurred, path does not exist or cannot be accessed + return false + } + + // Check if the path is a directory + return info.Mode().IsDir() +} + +func jsonFromObject(obj any) ([]byte, error) { + b, err := yamlv3.Marshal(obj) + if err != nil { + return nil, err + } + return yaml.YAMLToJSON(b) +} + +// inserts Backstage parameters at the specified path. Path is specified in the same format as jq. +func insertAt(ctx context.Context, input insertAtInput) (any, error) { + b, err := os.ReadFile(input.templatePath) + if err != nil { + return nil, err + } + var targetTemplate map[string]any + err = yaml.Unmarshal(b, &targetTemplate) + if err != nil { + return nil, err + } + jqProp, err := jsonFromObject(input.properties) + if err != nil { + return nil, err + } + + var sb strings.Builder + // update the properties field. merge (*) then assign (=) + sb.WriteString(fmt.Sprintf("%s.properties = %s.properties * %s", input.jqPathExpression, input.jqPathExpression, string(jqProp))) + if len(input.required) > 0 { + jqReq, err := jsonFromObject(input.required) + if err != nil { + return nil, err + } + // update the required field by feeding the new query the output from previous step (|) + sb.WriteString("| ") + sb.WriteString(fmt.Sprintf("%s.required = (%s.required + %s)", input.jqPathExpression, input.jqPathExpression, string(jqReq))) + } + query, err := gojq.Parse(sb.String()) + if err != nil { + log.Fatalln(err) + } + iter := query.RunWithContext(ctx, targetTemplate) + v, _ := iter.Next() + if err, ok := v.(error); ok { + log.Fatalln(err) + } + return v, nil +} + +func writeOutput(content any, path string) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return err + } + enc := yamlv3.NewEncoder(f) + defer enc.Close() + enc.SetIndent(2) + return enc.Encode(content) +} diff --git a/pkg/models/structs.go b/pkg/models/structs.go index 0d4bf5e..12ffc4b 100644 --- a/pkg/models/structs.go +++ b/pkg/models/structs.go @@ -79,3 +79,19 @@ type Template struct { } `yaml:"steps"` } `yaml:"spec"` } + +type BackstageParamFields struct { + Title string `yaml:",omitempty"` + Type string + Description string `yaml:",omitempty"` + Default any `yaml:",omitempty"` + Items *BackstageParamFields `yaml:",omitempty"` + UIWidget string `yaml:"ui:widget,omitempty"` + Properties map[string]*BackstageParamFields `yaml:"UiWidget,omitempty"` + AdditionalProperties *AdditionalProperties `yaml:"additionalProperties,omitempty"` + UniqueItems *bool `yaml:",omitempty"` // This does not guarantee a set. Works for primitives only. +} + +type AdditionalProperties struct { // technically any but for our case, it should be a type: string + Type string `yaml:",omitempty"` +} From 108b5f759fa687bb4d5dc76486a6b8e5597c6de4 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Wed, 9 Aug 2023 16:41:18 -0700 Subject: [PATCH 02/15] close file --- pkg/cmd/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go index 06f12ca..3022ee3 100644 --- a/pkg/cmd/utils.go +++ b/pkg/cmd/utils.go @@ -93,6 +93,7 @@ func writeOutput(content any, path string) error { if err != nil { return err } + defer f.Close() enc := yamlv3.NewEncoder(f) defer enc.Close() enc.SetIndent(2) From 025ff083e0712596f640f4c9eeeb1f343f0e3d32 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Wed, 9 Aug 2023 16:44:07 -0700 Subject: [PATCH 03/15] use yaml matcher --- pkg/cmd/terraform_test.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pkg/cmd/terraform_test.go b/pkg/cmd/terraform_test.go index 12c740c..0f6a779 100644 --- a/pkg/cmd/terraform_test.go +++ b/pkg/cmd/terraform_test.go @@ -4,12 +4,11 @@ import ( "context" "fmt" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" - "os" "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) var _ = Describe("Terraform Template", func() { @@ -66,15 +65,9 @@ var _ = Describe("Terraform Template", func() { It("should create the template file with properties merged", func() { generatedData, err := os.ReadFile(fmt.Sprintf("%s/input.yaml", outputDir)) Expect(err).NotTo(HaveOccurred()) - expectedData, err := os.ReadFile(expectedTemplateFile) - - var generated map[string]any - err = yaml.Unmarshal(generatedData, &generated) - var expected map[string]any - err = yaml.Unmarshal(expectedData, &expected) Expect(err).NotTo(HaveOccurred()) - Expect(generated).To(Equal(expected)) + Expect(generatedData).To(MatchYAML(expectedData)) }) It("should create the template file with properties merged and requirements updated", func() { From e5d17e93926fe53b07639ba0ec9f92290125a76b Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Fri, 11 Aug 2023 15:27:06 -0700 Subject: [PATCH 04/15] consolidate cmd flags --- pkg/cmd/crd.go | 1 - pkg/cmd/template.go | 3 +++ pkg/cmd/terraform.go | 5 +---- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index 63ed8f9..0b89c4d 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -34,7 +34,6 @@ var ( func init() { templateCmd.AddCommand(crdCmd) - crdCmd.Flags().StringVarP(&templatePath, "templatePath", "t", "scaffolding/template.yaml", "path to the template to be augmented with backstage info") crdCmd.Flags().StringArrayVarP(&verifiers, "verifier", "v", []string{}, "list of verifiers to test the resource against") crdCmd.Flags().BoolVarP(&namespaced, "namespaced", "n", false, "whether or not resources are namespaced") diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index a028fae..a73c5c0 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -13,6 +13,9 @@ func init() { rootCmd.AddCommand(templateCmd) templateCmd.PersistentFlags().StringVarP(&inputDir, "inputDir", "i", "", "input directory for CRDs and XRDs to be templatized") templateCmd.PersistentFlags().StringVarP(&outputDir, "outputDir", "o", "", "output directory for backstage templates to be stored in") + templateCmd.PersistentFlags().StringVarP(&templatePath, "templatePath", "t", "scaffolding/template.yaml", "path to the template to be augmented with backstage info") + templateCmd.PersistentFlags().Uint32Var(&depth, "depth", 2, "depth from given directory to search for TF modules or CRDs") + templateCmd.PersistentFlags().StringVarP(&insertionPoint, "insertAt", "p", "", "jq path within the template to insert backstage info") templateCmd.MarkFlagRequired("inputDir") templateCmd.MarkFlagRequired("outputDir") diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index 8172876..afaecd1 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -31,10 +31,7 @@ var ( ) func init() { - tfCmd.Flags().Uint32Var(&depth, "depth", 2, "depth from given directory to search for TF modules") - tfCmd.Flags().StringVarP(&templatePath, "templatePath", "t", "scaffolding/template.yaml", "path to the template to be augmented with backstage info") - tfCmd.Flags().StringVarP(&insertionPoint, "insertAt", "p", "", "jq path within the template to insert backstage info") - crdCmd.AddCommand(tfCmd) + templateCmd.AddCommand(tfCmd) } func tfE(cmd *cobra.Command, args []string) error { From 10c7322311fad27c95728333996d1b67005a5847 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Fri, 11 Aug 2023 15:45:17 -0700 Subject: [PATCH 05/15] document tf cmd behaviour --- pkg/cmd/terraform.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index afaecd1..3c87f10 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -18,6 +18,11 @@ var ( tfCmd = &cobra.Command{ Use: "tf", Short: "Generate backstage templates from Terraform variables", + Long: "Generate backstage templates by walking the given input directory, find TF modules," + + "then create output file per module.\n" + + "If the templatePath and insertionPoint flags are set, generated objects are merged into the given template at given insertion point.\n" + + "Otherwise a yaml file with two keys are generated. The properties key contains the generated form input. " + + "The required key contains the TF variable names that do not have defaults.", PreRunE: func(cmd *cobra.Command, args []string) error { if !isDirectory(inputDir) { return errors.New("inputDir and ouputDir entries need to be directories") From 202bec4e5eb4708864cc63d99f8935b399552ab1 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Fri, 11 Aug 2023 18:00:32 -0700 Subject: [PATCH 06/15] refactor file finders --- pkg/cmd/crd.go | 44 +++++++++++++++++++++++++++----------------- pkg/cmd/terraform.go | 40 ++++++++++++++++++++++++++++------------ pkg/cmd/utils.go | 23 +++++++++++++++++++++++ 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index 0b89c4d..af73ca6 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -79,7 +79,10 @@ func Crd( verifiers []string, namespaced bool, templateName, templateTitle, templateDescription string, ) error { - defs := defs(inputDir, 0) + defs, err := getDefs(inputDir, 0) + if err != nil { + return err + } output, err := writeSchema( stdout, stderr, @@ -107,26 +110,33 @@ func Crd( return nil } -func defs(dir string, depth int) []string { - if depth > 2 { - return nil +func getDefs(inputDir string, currentDepth uint32) ([]string, error) { + if currentDepth > depth { + return nil, nil } + out, err := getRelevantFiles(inputDir, currentDepth, findDefs) + if err != nil { + return nil, err + } + return out, err +} - var out []string - base, _ := filepath.Abs(dir) - files, _ := os.ReadDir(base) - for _, file := range files { - f := filepath.Join(base, file.Name()) - stat, _ := os.Stat(f) - if stat.IsDir() { - out = append(out, defs(f, depth+1)...) - continue +func findDefs(file os.DirEntry, currentDepth uint32, base string) ([]string, error) { + a := file.Name() + fmt.Println(a) + f := filepath.Join(base, file.Name()) + stat, err := os.Stat(f) + if err != nil { + return nil, err + } + if stat.IsDir() { + df, err := getDefs(f, currentDepth+1) + if err != nil { + return nil, err } - - out = append(out, f) + return df, nil } - - return out + return []string{f}, nil } func writeSchema(stdout, stderr io.Writer, outputDir string, defs []string) (cmdOutput, error) { diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index 3c87f10..d966774 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -44,7 +44,10 @@ func tfE(cmd *cobra.Command, args []string) error { } func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string) error { - mods := getModules(inputDir, 0) + mods, err := getModules(inputDir, 0) + if err != nil { + return err + } if len(mods) == 0 { return fmt.Errorf("could not find any TF modules in given directorr: %s", inputDir) } @@ -97,22 +100,34 @@ func handleOutput(ctx context.Context, outputFile, templatePath, insertionPoint return writeOutput(t, outputFile) } -func getModules(inputDir string, currentDepth uint32) []string { +func getModules(inputDir string, currentDepth uint32) ([]string, error) { if currentDepth > depth { - return nil + return nil, nil } if tfconfig.IsModuleDir(inputDir) { - return []string{inputDir} + return []string{inputDir}, nil } - base, _ := filepath.Abs(inputDir) - files, _ := os.ReadDir(base) - out := make([]string, 1) - for _, file := range files { - f := filepath.Join(base, file.Name()) - mods := getModules(f, currentDepth+1) - out = append(out, mods...) + out, err := getRelevantFiles(inputDir, currentDepth, findModule) + if err != nil { + return nil, err } - return out + return out, nil +} + +func findModule(file os.DirEntry, currentDepth uint32, base string) ([]string, error) { + f := filepath.Join(base, file.Name()) + stat, err := os.Stat(f) + if err != nil { + return nil, err + } + if stat.IsDir() { + mods, err := getModules(f, currentDepth+1) + if err != nil { + return nil, err + } + return mods, nil + } + return nil, nil } func convertVariable(tfVar tfconfig.Variable) models.BackstageParamFields { @@ -194,6 +209,7 @@ func convertObject(tfVar tfconfig.Variable) models.BackstageParamFields { p := models.AdditionalProperties{Type: mapType(nestedType)} out.AdditionalProperties = &p // defaults for object type is broken in Backstage atm. In the UI, the default values cannot be removed. + // we will enable this once it's fixed in Backstage. //properties := convertObjectDefaults(tfVar) //if len(properties) > 0 { // out.Properties = properties diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go index 3022ee3..d07cc8d 100644 --- a/pkg/cmd/utils.go +++ b/pkg/cmd/utils.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "path/filepath" "strings" "github.com/cnoe-io/cnoe-cli/pkg/models" @@ -13,6 +14,8 @@ import ( "sigs.k8s.io/yaml" ) +type finder func(file os.DirEntry, currentDepth uint32, base string) ([]string, error) + type NotFoundError struct { Err error } @@ -99,3 +102,23 @@ func writeOutput(content any, path string) error { enc.SetIndent(2) return enc.Encode(content) } + +func getRelevantFiles(inputDir string, currentDepth uint32, f finder) ([]string, error) { + base, err := filepath.Abs(inputDir) + if err != nil { + return nil, err + } + files, err := os.ReadDir(base) + if err != nil { + return nil, err + } + out := make([]string, 0) + for _, file := range files { + o, err := f(file, currentDepth, base) + if err != nil { + return nil, err + } + out = append(out, o...) + } + return out, nil +} From 37bb5bb541fd36c3d67673980833e372f74812a7 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Fri, 11 Aug 2023 18:10:30 -0700 Subject: [PATCH 07/15] remove unnecessary lines --- pkg/cmd/crd.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index af73ca6..4aaa88d 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -118,12 +118,10 @@ func getDefs(inputDir string, currentDepth uint32) ([]string, error) { if err != nil { return nil, err } - return out, err + return out, nil } func findDefs(file os.DirEntry, currentDepth uint32, base string) ([]string, error) { - a := file.Name() - fmt.Println(a) f := filepath.Join(base, file.Name()) stat, err := os.Stat(f) if err != nil { From 3e9d3725f33f5bfaf7fc7be18743c9120bf3ba79 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Mon, 14 Aug 2023 12:19:44 -0700 Subject: [PATCH 08/15] add more tests --- .../terraform/invalid/input/variables.tf | 5 +++ .../valid/output/properties-require.yaml | 39 +++++++++++++++++++ pkg/cmd/terraform.go | 2 +- pkg/cmd/terraform_test.go | 29 ++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 pkg/cmd/fakes/terraform/invalid/input/variables.tf create mode 100644 pkg/cmd/fakes/terraform/valid/output/properties-require.yaml diff --git a/pkg/cmd/fakes/terraform/invalid/input/variables.tf b/pkg/cmd/fakes/terraform/invalid/input/variables.tf new file mode 100644 index 0000000..1bd60f6 --- /dev/null +++ b/pkg/cmd/fakes/terraform/invalid/input/variables.tf @@ -0,0 +1,5 @@ +variable "name" { + description = "Name of the VPC and EKS Cluster" + type = string + default = "invalid-configuration" + diff --git a/pkg/cmd/fakes/terraform/valid/output/properties-require.yaml b/pkg/cmd/fakes/terraform/valid/output/properties-require.yaml new file mode 100644 index 0000000..0a2d3f1 --- /dev/null +++ b/pkg/cmd/fakes/terraform/valid/output/properties-require.yaml @@ -0,0 +1,39 @@ +properties: + eks_cluster_version: + type: string + description: EKS Cluster version + default: "1.27" + name: + type: string + description: Name of the VPC and EKS Cluster + default: emr-eks-ack + private_subnets: + type: array + description: Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet + default: + - 10.1.0.0/17 + - 10.1.128.0/18 + items: + type: string + public_subnets: + type: array + description: Public Subnets CIDRs. 62 IPs per Subnet + default: + - 10.1.255.128/26 + - 10.1.255.192/26 + items: + type: string + region: + type: string + description: Region + tags: + title: tags + type: object + description: Default tags + additionalProperties: + type: string + vpc_cidr: + type: string + description: VPC CIDR + default: 10.1.0.0/16 +required: [region] diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index d966774..284d1d6 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -70,7 +70,7 @@ func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertion required = append(required, j) } } - filePath := filepath.Join(outputDir, fmt.Sprintf("%s.yaml", filepath.Base(inputDir))) + filePath := filepath.Join(outputDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) err := handleOutput(ctx, filePath, templatePath, insertionPoint, params, required) if err != nil { log.Println(err) diff --git a/pkg/cmd/terraform_test.go b/pkg/cmd/terraform_test.go index 0f6a779..2fde316 100644 --- a/pkg/cmd/terraform_test.go +++ b/pkg/cmd/terraform_test.go @@ -18,9 +18,11 @@ var _ = Describe("Terraform Template", func() { ) const ( + validInputRootDir = "./fakes/terraform/valid" inputDir = "./fakes/terraform/valid/input" inputDirWithRequire = "./fakes/terraform/valid/input-require" expectedPropertyFile = "./fakes/terraform/valid/output/properties.yaml" + expectedPropertyFileWithRequire = "./fakes/terraform/valid/output/properties-require.yaml" expectedTemplateFile = "./fakes/terraform/valid/output/full-template.yaml" expectedTemplateFileWithRequire = "./fakes/terraform/valid/output/full-template-require.yaml" targetTemplateFile = "./fakes/template/input-template.yaml" @@ -93,4 +95,31 @@ var _ = Describe("Terraform Template", func() { Expect(generatedData).To(MatchYAML(expectedData)) }) }) + + Context("with a root directory specified", func() { + BeforeEach(func() { + err := terraform(context.Background(), validInputRootDir, outputDir, "", "") + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create properties files with properties merged and requirements updated", func() { + generatedInputData, err := os.ReadFile(fmt.Sprintf("%s/input.yaml", outputDir)) + Expect(err).NotTo(HaveOccurred()) + generatedInputRequireData, err := os.ReadFile(fmt.Sprintf("%s/input-require.yaml", outputDir)) + Expect(err).NotTo(HaveOccurred()) + expectedInputData, err := os.ReadFile(expectedPropertyFile) + Expect(err).NotTo(HaveOccurred()) + expectedInputRequireData, err := os.ReadFile(expectedPropertyFileWithRequire) + Expect(err).NotTo(HaveOccurred()) + Expect(generatedInputData).To(MatchYAML(expectedInputData)) + Expect(generatedInputRequireData).To(MatchYAML(expectedInputRequireData)) + }) + }) + + Context("with an invalid input and no target template specified", func() { + It("should return an error", func() { + err := terraform(context.Background(), "./fakes/terraform/invalid", outputDir, "", "") + Expect(err).Should(HaveOccurred()) + }) + }) }) From 2e99eb77e8ea4f33c5a2155609bbbd2cbe1cd407 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Mon, 14 Aug 2023 16:34:03 -0700 Subject: [PATCH 09/15] refactor crd command to use util functions where possible --- pkg/cmd/crd.go | 311 +++++++++--------- pkg/cmd/crd_test.go | 100 +++--- .../input}/invalid-input-resource.yaml | 0 pkg/cmd/fakes/crd/valid/input/cdn.yaml | 73 ++++ pkg/cmd/fakes/crd/valid/input/service.yaml | 11 + .../valid/input/sparkapp.yaml} | 0 .../full-template-awsblueprints.io.xcdn.yaml | 74 +++++ .../crd/valid/output/full-template-oneof.yaml | 37 +++ ...sparkoperator.k8s.io.sparkapplication.yaml | 63 ++++ .../properties-awsblueprints.io.xcdn.yaml | 48 +++ ...parkoperator.k8s.io.sparkapplication.yaml} | 11 +- .../valid/output/full-template-oneof.yaml | 37 +++ pkg/cmd/template.go | 29 +- pkg/cmd/terraform.go | 73 ++-- pkg/cmd/terraform_test.go | 36 +- pkg/cmd/utils.go | 100 +++++- pkg/models/structs.go | 1 + 17 files changed, 748 insertions(+), 256 deletions(-) rename pkg/cmd/fakes/{invalid-in-resource => crd/invalid/input}/invalid-input-resource.yaml (100%) create mode 100644 pkg/cmd/fakes/crd/valid/input/cdn.yaml create mode 100644 pkg/cmd/fakes/crd/valid/input/service.yaml rename pkg/cmd/fakes/{in-resource/input-resource.yaml => crd/valid/input/sparkapp.yaml} (100%) create mode 100644 pkg/cmd/fakes/crd/valid/output/full-template-awsblueprints.io.xcdn.yaml create mode 100644 pkg/cmd/fakes/crd/valid/output/full-template-oneof.yaml create mode 100644 pkg/cmd/fakes/crd/valid/output/full-template-sparkoperator.k8s.io.sparkapplication.yaml create mode 100644 pkg/cmd/fakes/crd/valid/output/properties-awsblueprints.io.xcdn.yaml rename pkg/cmd/fakes/{out-resource/output-resource.yaml => crd/valid/output/properties-sparkoperator.k8s.io.sparkapplication.yaml} (84%) create mode 100644 pkg/cmd/fakes/terraform/valid/output/full-template-oneof.yaml diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index 4aaa88d..93b5249 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -1,9 +1,10 @@ package cmd import ( + "context" "errors" "fmt" - "io" + "log" "os" "path/filepath" "strings" @@ -21,11 +22,7 @@ const ( ) var ( - inputDir string - outputDir string - templatePath string - verifiers []string - namespaced bool + verifiers []string templateName string templateTitle string @@ -35,27 +32,19 @@ var ( func init() { templateCmd.AddCommand(crdCmd) crdCmd.Flags().StringArrayVarP(&verifiers, "verifier", "v", []string{}, "list of verifiers to test the resource against") - crdCmd.Flags().BoolVarP(&namespaced, "namespaced", "n", false, "whether or not resources are namespaced") crdCmd.Flags().StringVarP(&templateName, "templateName", "", "", "sets the name of the template") crdCmd.Flags().StringVarP(&templateTitle, "templateTitle", "", "", "sets the title of the template") crdCmd.Flags().StringVarP(&templateDescription, "templateDescription", "", "", "sets the description of the template") - - crdCmd.MarkFlagRequired("templatePath") } var ( crdCmd = &cobra.Command{ - Use: "crd", - Short: "Generate backstage templates from CRD/XRD", - Long: `Generate backstage templates from supplied CRD and XRD definitions`, - PreRunE: func(cmd *cobra.Command, args []string) error { - if !isDirectory(inputDir) || !isDirectory(outputDir) { - return errors.New("inputDir and ouputDir entries need to be directories") - } - return nil - }, - RunE: crd, + Use: "crd", + Short: "Generate backstage templates from CRD/XRD", + Long: `Generate backstage templates from supplied CRD and XRD definitions`, + PreRunE: templatePreRunE, + RunE: crd, } ) @@ -66,43 +55,51 @@ type cmdOutput struct { func crd(cmd *cobra.Command, args []string) error { return Crd( - cmd.OutOrStdout(), cmd.OutOrStderr(), - inputDir, outputDir, templatePath, - verifiers, namespaced, + cmd.Context(), + inputDir, outputDir, templatePath, insertionPoint, + verifiers, useOneOf, templateName, templateTitle, templateDescription, ) } func Crd( - stdout, stderr io.Writer, - inputDir, outputDir, templatePath string, - verifiers []string, namespaced bool, + ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, + verifiers []string, useOneOf bool, templateName, templateTitle, templateDescription string, ) error { - defs, err := getDefs(inputDir, 0) + inDir, outDir, template, err := prepDirectories(inputDir, outputDir, templatePath, useOneOf) if err != nil { return err } - - output, err := writeSchema( - stdout, stderr, - outputDir, - defs, - ) + defs, err := getDefs(inDir, 0) if err != nil { return err } - - err = writeToTemplate( - stdout, stderr, - templatePath, - outputDir, - output.Resources, 0, - templateName, - templateTitle, - templateDescription, + log.Printf("processing %d definitions", len(defs)) + + schemaDir := outDir + if useOneOf { + schemaDir = filepath.Join(outDir, defDir) + output, err := writeSchema( + ctx, + schemaDir, + "", + "", + defs, + ) + if err != nil { + return err + } + templateFile := filepath.Join(outDir, "template.yaml") + return writeOneOf(ctx, templateFile, template, insertionPoint, output.Templates) + } + _, err = writeSchema( + ctx, + schemaDir, + insertionPoint, + template, + defs, ) - if err != nil { return err } @@ -137,125 +134,46 @@ func findDefs(file os.DirEntry, currentDepth uint32, base string) ([]string, err return []string{f}, nil } -func writeSchema(stdout, stderr io.Writer, outputDir string, defs []string) (cmdOutput, error) { +func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile string, defs []string) (cmdOutput, error) { out := cmdOutput{ Templates: make([]string, 0), Resources: make([]string, 0), } - templateOutputDir := fmt.Sprintf("%s/%s", outputDir, defDir) - _, err := os.Stat(templateOutputDir) - if os.IsNotExist(err) { - // Directory doesn't exist, so create it - err := os.MkdirAll(templateOutputDir, 0755) - if err != nil { - return cmdOutput{}, err - } - fmt.Fprintf(stdout, "Directory created successfully!") - } else if err != nil { - return cmdOutput{}, err - } - for _, def := range defs { - data, err := os.ReadFile(def) - if err != nil { - continue - } - - var doc models.Definition - err = yaml.Unmarshal(data, &doc) + converted, resourceName, err := convert(def) if err != nil { - fmt.Printf("failed to read %s. This file will be excluded. %s", def, err) - continue - } - - if !isXRD(doc) && !isCRD(doc) { - continue - } - - fmt.Fprintf(stdout, "foud: %s\n", def) - var resourceName string - if doc.Spec.ClaimNames != nil { - resourceName = doc.Spec.ClaimNames.Kind - } else { - resourceName = fmt.Sprintf("%s.%s", doc.Spec.Group, doc.Spec.Names.Kind) + var e NotSupported + if errors.As(err, &e) { + continue + } + return cmdOutput{}, err } - - var value map[string]interface{} - - v := doc.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] - if v == nil { - value = doc.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties - } else { - value, err = ConvertMap(v) + filename := filepath.Join(outputDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) + if templateFile != "" && insertionPoint != "" { + input := insertAtInput{ + templatePath: templateFile, + jqPathExpression: insertionPoint, + } + props := converted.(map[string]any) + if v, reqOk := props["required"]; reqOk { + if reqs, ok := v.([]string); ok { + input.required = reqs + } + } + input.fields = props + t, err := insertAt(ctx, input) if err != nil { return cmdOutput{}, err } + converted = t } - - obj := &unstructured.Unstructured{ - Object: make(map[string]interface{}, 0), - } - unstructured.SetNestedSlice(obj.Object, ConvertSlice([]string{resourceName}), "properties", "resources", "enum") - unstructured.SetNestedMap(obj.Object, value, "properties", "config") - unstructured.SetNestedField(obj.Object, fmt.Sprintf("%s configuration options", resourceName), "properties", "config", "title") - - // setting GVK for the resource - if len(doc.Spec.Versions) > 0 { - unstructured.SetNestedMap(obj.Object, map[string]interface{}{ - "type": "string", - "description": "APIVersion for the resource", - "default": fmt.Sprintf("%s/%s", doc.Spec.Group, doc.Spec.Versions[0].Name), - }, - "properties", "apiVersion") - unstructured.SetNestedMap(obj.Object, map[string]interface{}{ - "type": "string", - "description": "Kind for the resource", - "default": doc.Spec.Names.Kind, - }, - "properties", "kind") - } - - // add a property to define the namespace for the resource - if namespaced { - unstructured.SetNestedMap(obj.Object, map[string]interface{}{ - "type": "string", - "description": "Namespace for the resource", - "namespace": "default", - }, - "properties", "namespace") - } - - // add verifiers to the resource - if len(verifiers) > 0 { - var convertedVerifiers []interface{} = make([]interface{}, len(verifiers)) - for i, v := range verifiers { - convertedVerifiers[i] = v - } - - unstructured.SetNestedMap(obj.Object, map[string]interface{}{ - "type": "array", - "description": "verifiers to be used against the resource", - "items": map[string]interface{}{"type": "string"}, - "default": convertedVerifiers, - }, - "properties", "verifiers") - } - - wrapperData, err := yaml.Marshal(obj.Object) - if err != nil { - fmt.Fprintf(stdout, "failed %s: %s \n", def, err.Error()) - continue - } - - template := fmt.Sprintf("%s/%s.yaml", templateOutputDir, strings.ToLower(resourceName)) - err = os.WriteFile(template, []byte(wrapperData), 0644) + err = writeOutput(converted, filename) if err != nil { - fmt.Fprintf(stdout, "failed %s: %s \n", def, err.Error()) - continue + log.Printf("failed to write %s: %s \n", def, err.Error()) + return cmdOutput{}, err } - - out.Templates = append(out.Templates, template) + out.Templates = append(out.Templates, filename) out.Resources = append(out.Resources, resourceName) } @@ -263,7 +181,6 @@ func writeSchema(stdout, stderr io.Writer, outputDir string, defs []string) (cmd } func writeToTemplate( - stdout, stderr io.Writer, templateFile string, outputPath string, identifiedResources []string, position int, templateName, templateTitle, templateDescription string, ) error { @@ -296,14 +213,6 @@ func writeToTemplate( } `yaml:"resources,omitempty"` }{} - resources := struct { - Type string `yaml:"type"` - Enum []string `yaml:"enum"` - }{ - Type: "string", - Enum: identifiedResources, - } - for _, r := range identifiedResources { dependencies.Resources.OneOf = append(dependencies.Resources.OneOf, map[string]interface{}{ "$yaml": fmt.Sprintf("resources/%s.yaml", strings.ToLower(r)), @@ -314,7 +223,6 @@ func writeToTemplate( return errors.New("not the right template or input format") } - doc.Spec.Parameters[position].Properties["resources"] = resources doc.Spec.Parameters[position].Dependencies = dependencies outputData, err := yaml.Marshal(&doc) @@ -327,10 +235,97 @@ func writeToTemplate( return err } - fmt.Fprintf(stdout, "Template successfully written.") return nil } +func convert(def string) (any, string, error) { + data, err := os.ReadFile(def) + if err != nil { + return nil, "", err + } + var doc models.Definition + err = yaml.Unmarshal(data, &doc) + if err != nil { + log.Printf("failed to read %s. This file will be excluded. %s", def, err) + return nil, "", NotSupported{ + fmt.Errorf("%s is not a kubernetes file", def), + } + } + + if !isXRD(doc) && !isCRD(doc) { + return nil, "", NotSupported{ + fmt.Errorf("%s is not a CRD or XRD", def), + } + } + var resourceName string + if doc.Spec.ClaimNames != nil { + resourceName = doc.Spec.ClaimNames.Kind + } else { + resourceName = fmt.Sprintf("%s.%s", doc.Spec.Group, doc.Spec.Names.Kind) + } + + var value map[string]interface{} + + v := doc.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + if v == nil { + value = doc.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties + } else { + value, err = ConvertMap(v) + if err != nil { + return cmdOutput{}, "", err + } + } + + obj := &unstructured.Unstructured{ + Object: make(map[string]interface{}, 0), + } + unstructured.SetNestedMap(obj.Object, value, "properties", "config") + unstructured.SetNestedField(obj.Object, fmt.Sprintf("%s configuration options", resourceName), "properties", "config", "title") + + // setting GVK for the resource + if len(doc.Spec.Versions) > 0 { + unstructured.SetNestedMap(obj.Object, map[string]interface{}{ + "type": "string", + "description": "APIVersion for the resource", + "default": fmt.Sprintf("%s/%s", doc.Spec.Group, doc.Spec.Versions[0].Name), + }, + "properties", "apiVersion") + unstructured.SetNestedMap(obj.Object, map[string]interface{}{ + "type": "string", + "description": "Kind for the resource", + "default": doc.Spec.Names.Kind, + }, + "properties", "kind") + } + + // add a property to define the namespace for the resource + if doc.Spec.Scope == "Namespaced" { + unstructured.SetNestedMap(obj.Object, map[string]interface{}{ + "type": "string", + "description": "Namespace for the resource", + "namespace": "default", + }, + "properties", "namespace") + } + + // add verifiers to the resource + if len(verifiers) > 0 { + var convertedVerifiers []interface{} = make([]interface{}, len(verifiers)) + for i, v := range verifiers { + convertedVerifiers[i] = v + } + + unstructured.SetNestedMap(obj.Object, map[string]interface{}{ + "type": "array", + "description": "verifiers to be used against the resource", + "items": map[string]interface{}{"type": "string"}, + "default": convertedVerifiers, + }, + "properties", "verifiers") + } + return obj.Object, resourceName, nil +} + func ConvertSlice(strSlice []string) []interface{} { var ifaceSlice []interface{} for _, s := range strSlice { diff --git a/pkg/cmd/crd_test.go b/pkg/cmd/crd_test.go index eead1df..7da2e9b 100644 --- a/pkg/cmd/crd_test.go +++ b/pkg/cmd/crd_test.go @@ -1,27 +1,20 @@ package cmd_test import ( + "context" "fmt" - "path" + "os" + "path/filepath" "github.com/cnoe-io/cnoe-cli/pkg/cmd" - "github.com/cnoe-io/cnoe-cli/pkg/models" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gbytes" - "gopkg.in/yaml.v3" - - "os" - "path/filepath" ) -var _ = Describe("Template", func() { +var _ = Describe("Template CRDs", func() { var ( tempDir string outputDir string - - stdout *gbytes.Buffer - stderr *gbytes.Buffer ) const ( @@ -29,11 +22,11 @@ var _ = Describe("Template", func() { templateTitle = "test-title" templateDescription = "test-description" - inputDir = "./fakes/in-resource" - invalidInputDir = "./fakes/invalid-in-resource" + inputDir = "./fakes/crd/valid/input" + validOutputDir = "./fakes/crd/valid/output" + invalidInputDir = "./fakes/crd/invalid/input" templateFile = "./fakes/template/input-template.yaml" - expectedTemplateFile = "./fakes/template/output-template.yaml" - expectedResourceFile = "./fakes/out-resource/output-resource.yaml" + expectedTemplateFile = "./fakes/crd/valid/output/full-template-oneof.yaml" ) BeforeEach(func() { @@ -44,9 +37,6 @@ var _ = Describe("Template", func() { outputDir = filepath.Join(tempDir, "output") err = os.Mkdir(outputDir, 0755) Expect(err).NotTo(HaveOccurred()) - - stdout = gbytes.NewBuffer() - stderr = gbytes.NewBuffer() }) AfterEach(func() { @@ -54,10 +44,10 @@ var _ = Describe("Template", func() { Expect(err).NotTo(HaveOccurred()) }) - Context("with valid input", func() { + Context("with valid input with oneof", func() { BeforeEach(func() { - err := cmd.Crd(stdout, stderr, inputDir, outputDir, templateFile, - []string{}, false, templateName, templateTitle, templateDescription, + err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", + []string{}, true, templateName, templateTitle, templateDescription, ) Expect(err).NotTo(HaveOccurred()) }) @@ -65,58 +55,66 @@ var _ = Describe("Template", func() { It("should create the template files for valid definitions", func() { expectedTemplateData, err := os.ReadFile(expectedTemplateFile) Expect(err).NotTo(HaveOccurred()) - - var expectedTemplate models.Template - err = yaml.Unmarshal(expectedTemplateData, &expectedTemplate) - Expect(err).NotTo(HaveOccurred()) - generatedTemplateData, err := os.ReadFile(fmt.Sprintf("%s/%s", outputDir, "template.yaml")) Expect(err).NotTo(HaveOccurred()) - var generatedTemplate models.Template - err = yaml.Unmarshal(generatedTemplateData, &generatedTemplate) - Expect(err).NotTo(HaveOccurred()) - Expect(generatedTemplate).To(Equal(expectedTemplate)) + Expect(expectedTemplateData).To(MatchYAML(generatedTemplateData)) }) It("should create valid resources", func() { - resourceDir := fmt.Sprintf("%s/%s", outputDir, "resources") + resourceDir := filepath.Join(outputDir, "resources") files, err := os.ReadDir(resourceDir) Expect(err).NotTo(HaveOccurred()) - Expect(len(files)).To(Equal(1)) - - filePath := path.Join(resourceDir, files[0].Name()) - Expect(err).NotTo(HaveOccurred()) - generatedResourceData, err := os.ReadFile(filePath) - Expect(err).NotTo(HaveOccurred()) + Expect(len(files)).To(Equal(2)) + + for i := range files { + filePath := filepath.Join(resourceDir, files[i].Name()) + genrated, err := os.ReadFile(filePath) + Expect(err).NotTo(HaveOccurred()) + propFile := fmt.Sprintf("properties-%s", files[i].Name()) + expected, err := os.ReadFile(filepath.Join(validOutputDir, propFile)) + Expect(err).NotTo(HaveOccurred()) + Expect(genrated).To(MatchYAML(expected)) + } + }) + }) - var generatedResource models.Definition - err = yaml.Unmarshal(generatedResourceData, &generatedResource) + Context("with valid input and specify template file and jq path", func() { + BeforeEach(func() { + err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", + []string{}, false, templateName, templateTitle, templateDescription, + ) Expect(err).NotTo(HaveOccurred()) + }) - expectedResourceData, err := os.ReadFile(expectedResourceFile) + It("should create valid backstage template for each definition", func() { + files, err := os.ReadDir(outputDir) Expect(err).NotTo(HaveOccurred()) - var expectedResource models.Definition - err = yaml.Unmarshal(expectedResourceData, &expectedResource) - Expect(err).NotTo(HaveOccurred()) - - Expect(generatedResource).To(Equal(expectedResource)) + Expect(len(files)).To(Equal(2)) + for i := range files { + filePath := filepath.Join(outputDir, files[i].Name()) + generated, err := os.ReadFile(filePath) + Expect(err).NotTo(HaveOccurred()) + propFile := fmt.Sprintf("full-template-%s", files[i].Name()) + expected, err := os.ReadFile(filepath.Join(validOutputDir, propFile)) + Expect(err).NotTo(HaveOccurred()) + Expect(generated).To(MatchYAML(expected)) + } }) }) - Context("with invalid input files", func() { + Context("with invalid input only", func() { BeforeEach(func() { - err := cmd.Crd(stdout, stderr, invalidInputDir, outputDir, templateFile, + err := cmd.Crd(context.Background(), invalidInputDir, outputDir, "", "", []string{}, false, templateName, templateTitle, templateDescription, ) Expect(err).NotTo(HaveOccurred()) }) - It("should create the template files for valid definitions only", func() { - resourceDir := fmt.Sprintf("%s/%s", outputDir, "resources") - files, err := os.ReadDir(resourceDir) + It("should not create any files", func() { + files, err := os.ReadDir(outputDir) Expect(err).NotTo(HaveOccurred()) - Expect(len(files)).To(Equal(1)) + Expect(len(files)).To(Equal(0)) }) }) }) diff --git a/pkg/cmd/fakes/invalid-in-resource/invalid-input-resource.yaml b/pkg/cmd/fakes/crd/invalid/input/invalid-input-resource.yaml similarity index 100% rename from pkg/cmd/fakes/invalid-in-resource/invalid-input-resource.yaml rename to pkg/cmd/fakes/crd/invalid/input/invalid-input-resource.yaml diff --git a/pkg/cmd/fakes/crd/valid/input/cdn.yaml b/pkg/cmd/fakes/crd/valid/input/cdn.yaml new file mode 100644 index 0000000..23dca32 --- /dev/null +++ b/pkg/cmd/fakes/crd/valid/input/cdn.yaml @@ -0,0 +1,73 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xcdns.awsblueprints.io +spec: + claimNames: + kind: CDN + plural: cdns + group: awsblueprints.io + names: + kind: XCDN + plural: xcdns + connectionSecretKeys: + - region + - bucket-name + - s3-put-policy + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + properties: + spec: + properties: + resourceConfig: + description: ResourceConfig defines general properties of this AWS + resource. + properties: + deletionPolicy: + description: Defaults to Delete + enum: + - Delete + - Orphan + type: string + name: + description: Set the name of this resource in AWS to the value + provided by this field. + type: string + providerConfigName: + type: string + region: + type: string + tags: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + required: + - providerConfigName + - region + - tags + type: object + required: + - resourceConfig + type: object + status: + properties: + bucketName: + type: string + bucketArn: + type: string + oaiId: + type: string + type: object + type: object diff --git a/pkg/cmd/fakes/crd/valid/input/service.yaml b/pkg/cmd/fakes/crd/valid/input/service.yaml new file mode 100644 index 0000000..afcf117 --- /dev/null +++ b/pkg/cmd/fakes/crd/valid/input/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: my-service +spec: + selector: + app.kubernetes.io/name: MyApp + ports: + - protocol: TCP + port: 80 + targetPort: 9376 diff --git a/pkg/cmd/fakes/in-resource/input-resource.yaml b/pkg/cmd/fakes/crd/valid/input/sparkapp.yaml similarity index 100% rename from pkg/cmd/fakes/in-resource/input-resource.yaml rename to pkg/cmd/fakes/crd/valid/input/sparkapp.yaml diff --git a/pkg/cmd/fakes/crd/valid/output/full-template-awsblueprints.io.xcdn.yaml b/pkg/cmd/fakes/crd/valid/output/full-template-awsblueprints.io.xcdn.yaml new file mode 100644 index 0000000..233bcef --- /dev/null +++ b/pkg/cmd/fakes/crd/valid/output/full-template-awsblueprints.io.xcdn.yaml @@ -0,0 +1,74 @@ +apiVersion: scaffolder.backstage.io/v1beta3 +kind: Template +metadata: + description: Deploy Resource to Kubernetes + name: deploy-resources + title: Deploy Resources +spec: + owner: guest + parameters: + - description: Select a AWS resource to add to your repository. + properties: + apiVersion: + default: awsblueprints.io/v1alpha1 + description: APIVersion for the resource + type: string + config: + properties: + resourceConfig: + description: ResourceConfig defines general properties of this AWS resource. + properties: + deletionPolicy: + description: Defaults to Delete + enum: + - Delete + - Orphan + type: string + name: + description: Set the name of this resource in AWS to the value provided by this field. + type: string + providerConfigName: + type: string + region: + type: string + tags: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + required: + - providerConfigName + - region + - tags + type: object + required: + - resourceConfig + title: awsblueprints.io.XCDN configuration options + type: object + kind: + default: XCDN + description: Kind for the resource + type: string + name: + description: name of this resource. This will be the name of K8s object. + type: string + path: + default: kustomize/base + description: path to place this file into + type: string + required: + - awsResources + - name + title: Choose Resource + steps: + - action: cnoe:verify:dependency + id: verify + name: verify + type: service diff --git a/pkg/cmd/fakes/crd/valid/output/full-template-oneof.yaml b/pkg/cmd/fakes/crd/valid/output/full-template-oneof.yaml new file mode 100644 index 0000000..ea062bd --- /dev/null +++ b/pkg/cmd/fakes/crd/valid/output/full-template-oneof.yaml @@ -0,0 +1,37 @@ +apiVersion: scaffolder.backstage.io/v1beta3 +kind: Template +metadata: + description: Deploy Resource to Kubernetes + name: deploy-resources + title: Deploy Resources +spec: + owner: guest + parameters: + - dependencies: + resources: + oneOf: + - $yaml: resources/awsblueprints.io.xcdn.yaml + - $yaml: resources/sparkoperator.k8s.io.sparkapplication.yaml + description: Select a AWS resource to add to your repository. + properties: + name: + description: name of this resource. This will be the name of K8s object. + type: string + path: + default: kustomize/base + description: path to place this file into + type: string + resources: + enum: + - awsblueprints.io.xcdn + - sparkoperator.k8s.io.sparkapplication + type: string + required: + - awsResources + - name + title: Choose Resource + steps: + - action: cnoe:verify:dependency + id: verify + name: verify + type: service diff --git a/pkg/cmd/fakes/crd/valid/output/full-template-sparkoperator.k8s.io.sparkapplication.yaml b/pkg/cmd/fakes/crd/valid/output/full-template-sparkoperator.k8s.io.sparkapplication.yaml new file mode 100644 index 0000000..b08221b --- /dev/null +++ b/pkg/cmd/fakes/crd/valid/output/full-template-sparkoperator.k8s.io.sparkapplication.yaml @@ -0,0 +1,63 @@ +apiVersion: scaffolder.backstage.io/v1beta3 +kind: Template +metadata: + description: Deploy Resource to Kubernetes + name: deploy-resources + title: Deploy Resources +spec: + owner: guest + parameters: + - description: Select a AWS resource to add to your repository. + properties: + apiVersion: + default: sparkoperator.k8s.io/v1beta2 + description: APIVersion for the resource + type: string + config: + properties: + arguments: + items: + type: string + type: array + batchScheduler: + type: string + batchSchedulerOptions: + properties: + priorityClassName: + type: string + queue: + type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + title: sparkoperator.k8s.io.SparkApplication configuration options + kind: + default: SparkApplication + description: Kind for the resource + type: string + name: + description: name of this resource. This will be the name of K8s object. + type: string + namespace: + description: Namespace for the resource + namespace: default + type: string + path: + default: kustomize/base + description: path to place this file into + type: string + required: + - awsResources + - name + title: Choose Resource + steps: + - action: cnoe:verify:dependency + id: verify + name: verify + type: service diff --git a/pkg/cmd/fakes/crd/valid/output/properties-awsblueprints.io.xcdn.yaml b/pkg/cmd/fakes/crd/valid/output/properties-awsblueprints.io.xcdn.yaml new file mode 100644 index 0000000..e0f1c4d --- /dev/null +++ b/pkg/cmd/fakes/crd/valid/output/properties-awsblueprints.io.xcdn.yaml @@ -0,0 +1,48 @@ +properties: + apiVersion: + default: awsblueprints.io/v1alpha1 + description: APIVersion for the resource + type: string + config: + properties: + resourceConfig: + description: ResourceConfig defines general properties of this AWS resource. + properties: + deletionPolicy: + description: Defaults to Delete + enum: + - Delete + - Orphan + type: string + name: + description: Set the name of this resource in AWS to the value provided by this field. + type: string + providerConfigName: + type: string + region: + type: string + tags: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + required: + - providerConfigName + - region + - tags + type: object + required: + - resourceConfig + title: awsblueprints.io.XCDN configuration options + type: object + kind: + default: XCDN + description: Kind for the resource + type: string diff --git a/pkg/cmd/fakes/out-resource/output-resource.yaml b/pkg/cmd/fakes/crd/valid/output/properties-sparkoperator.k8s.io.sparkapplication.yaml similarity index 84% rename from pkg/cmd/fakes/out-resource/output-resource.yaml rename to pkg/cmd/fakes/crd/valid/output/properties-sparkoperator.k8s.io.sparkapplication.yaml index 9263254..9059ad6 100644 --- a/pkg/cmd/fakes/out-resource/output-resource.yaml +++ b/pkg/cmd/fakes/crd/valid/output/properties-sparkoperator.k8s.io.sparkapplication.yaml @@ -3,9 +3,6 @@ properties: default: sparkoperator.k8s.io/v1beta2 description: APIVersion for the resource type: string - awsResources: - enum: - - sparkoperator.k8s.io.SparkApplication config: properties: arguments: @@ -23,8 +20,8 @@ properties: resources: additionalProperties: anyOf: - - type: integer - - type: string + - type: integer + - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -34,3 +31,7 @@ properties: default: SparkApplication description: Kind for the resource type: string + namespace: + description: Namespace for the resource + namespace: default + type: string diff --git a/pkg/cmd/fakes/terraform/valid/output/full-template-oneof.yaml b/pkg/cmd/fakes/terraform/valid/output/full-template-oneof.yaml new file mode 100644 index 0000000..ab732b8 --- /dev/null +++ b/pkg/cmd/fakes/terraform/valid/output/full-template-oneof.yaml @@ -0,0 +1,37 @@ +apiVersion: scaffolder.backstage.io/v1beta3 +kind: Template +metadata: + name: deploy-resources + title: Deploy Resources + description: Deploy Resource to Kubernetes +spec: + owner: guest + type: service + parameters: + - title: Choose Resource + description: Select a AWS resource to add to your repository. + properties: + resources: + type: string + enum: + - input + - input-require + name: + description: name of this resource. This will be the name of K8s object. + type: string + path: + default: kustomize/base + description: path to place this file into + type: string + dependencies: + resources: + oneOf: + - $yaml: resources/input.yaml + - $yaml: resources/input-require.yaml + required: + - awsResources + - name + steps: + - id: verify + name: verify + action: cnoe:verify:dependency diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index a73c5c0..04a44a9 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -1,6 +1,8 @@ package cmd import ( + "errors" + "github.com/spf13/cobra" ) @@ -9,14 +11,39 @@ var templateCmd = &cobra.Command{ Short: "Generate Backstage templates", } +var ( + depth uint32 + insertionPoint string + inputDir string + outputDir string + templatePath string + useOneOf bool +) + func init() { rootCmd.AddCommand(templateCmd) templateCmd.PersistentFlags().StringVarP(&inputDir, "inputDir", "i", "", "input directory for CRDs and XRDs to be templatized") templateCmd.PersistentFlags().StringVarP(&outputDir, "outputDir", "o", "", "output directory for backstage templates to be stored in") - templateCmd.PersistentFlags().StringVarP(&templatePath, "templatePath", "t", "scaffolding/template.yaml", "path to the template to be augmented with backstage info") + templateCmd.PersistentFlags().StringVarP(&templatePath, "templatePath", "t", "", "path to the template to be augmented with backstage info") templateCmd.PersistentFlags().Uint32Var(&depth, "depth", 2, "depth from given directory to search for TF modules or CRDs") templateCmd.PersistentFlags().StringVarP(&insertionPoint, "insertAt", "p", "", "jq path within the template to insert backstage info") + templateCmd.PersistentFlags().BoolVarP(&useOneOf, "oneTemplate", "u", false, "if set to true, items are rendered as drop down items in the specified template") templateCmd.MarkFlagRequired("inputDir") templateCmd.MarkFlagRequired("outputDir") } + +func templatePreRunE(cmd *cobra.Command, args []string) error { + if !isDirectory(inputDir) { + return errors.New("inputDir must be a directory") + } + + if useOneOf && templatePath == "" && insertionPoint == "" { + return errors.New("templatePath and inertAt flags must be specified when using the oneTemplate flag") + } + + if insertionPoint != "" && templatePath == "" { + return errors.New("templatePath flag must be specified") + } + return nil +} diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index 284d1d6..6f371f0 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "errors" "fmt" "log" "os" @@ -23,16 +22,9 @@ var ( "If the templatePath and insertionPoint flags are set, generated objects are merged into the given template at given insertion point.\n" + "Otherwise a yaml file with two keys are generated. The properties key contains the generated form input. " + "The required key contains the TF variable names that do not have defaults.", - PreRunE: func(cmd *cobra.Command, args []string) error { - if !isDirectory(inputDir) { - return errors.New("inputDir and ouputDir entries need to be directories") - } - return nil - }, - RunE: tfE, + PreRunE: templatePreRunE, + RunE: tfE, } - depth uint32 - insertionPoint string ) func init() { @@ -40,52 +32,81 @@ func init() { } func tfE(cmd *cobra.Command, args []string) error { - return terraform(cmd.Context(), inputDir, outputDir, templatePath, insertionPoint) + return terraform(cmd.Context(), inputDir, outputDir, templatePath, insertionPoint, useOneOf) } -func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string) error { - mods, err := getModules(inputDir, 0) +func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, useOneOf bool) error { + inDir, outDir, template, err := prepDirectories(inputDir, outputDir, templatePath, useOneOf) + mods, err := getModules(inDir, 0) if err != nil { return err } if len(mods) == 0 { - return fmt.Errorf("could not find any TF modules in given directorr: %s", inputDir) + return fmt.Errorf("could not find any TF modules in given directorr: %s", inDir) } - + log.Printf("processing %d modules", len(mods)) for i := range mods { path := mods[i] + log.Printf("processing module at %s", path) mod, diag := tfconfig.LoadModule(path) if diag.HasErrors() { return diag.Err() } if len(mod.Variables) == 0 { - fmt.Println(fmt.Sprintf("module %s does not have variables", path)) + log.Printf(fmt.Sprintf("module %s does not have variables", path)) continue } params := make(map[string]models.BackstageParamFields) required := make([]string, 0) + log.Printf("converting %d variables", len(mod.Variables)) for j := range mod.Variables { params[j] = convertVariable(*mod.Variables[j]) if mod.Variables[j].Required { required = append(required, j) } } - filePath := filepath.Join(outputDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) - err := handleOutput(ctx, filePath, templatePath, insertionPoint, params, required) - if err != nil { - log.Println(err) + if useOneOf { + p := filepath.Join(outDir, defDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) + log.Printf("writing to %s", p) + err := handleOutput(ctx, p, "", "", params, required) + if err != nil { + return err + } + } else { + p := filepath.Join(outDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) + log.Printf("writing to %s", p) + err := handleOutput(ctx, p, template, insertionPoint, params, required) + if err != nil { + return err + } } } + if useOneOf { + templateFile := filepath.Join(outDir, "template.yaml") + resourceFileNames := make([]string, len(mods)) + for i := range mods { + resourceFileNames[i] = fmt.Sprintf("%s.yaml", mods[i]) + } + return writeOneOf(ctx, templateFile, template, insertionPoint, resourceFileNames) + } return nil } -func handleOutput(ctx context.Context, outputFile, templatePath, insertionPoint string, properties map[string]models.BackstageParamFields, required []string) error { - if templatePath != "" && insertionPoint != "" { +func handleOutput(ctx context.Context, outputFile, templateFile, insertionPoint string, properties map[string]models.BackstageParamFields, required []string) error { + props := make(map[string]any, len(properties)) + for k := range properties { + props[k] = properties[k] + } + if templateFile != "" && insertionPoint != "" { input := insertAtInput{ - templatePath: templatePath, + templatePath: templateFile, jqPathExpression: insertionPoint, - properties: properties, - required: required, + fields: map[string]any{ + "properties": props, + }, + } + if len(required) > 0 { + input.required = required } t, err := insertAt(ctx, input) if err != nil { @@ -94,7 +115,7 @@ func handleOutput(ctx context.Context, outputFile, templatePath, insertionPoint return writeOutput(t, outputFile) } t := map[string]any{ - "properties": properties, + "properties": props, "required": required, } return writeOutput(t, outputFile) diff --git a/pkg/cmd/terraform_test.go b/pkg/cmd/terraform_test.go index 2fde316..29f97af 100644 --- a/pkg/cmd/terraform_test.go +++ b/pkg/cmd/terraform_test.go @@ -45,7 +45,7 @@ var _ = Describe("Terraform Template", func() { Context("with valid input and no target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDir, outputDir, "", "") + err := terraform(context.Background(), inputDir, outputDir, "", "", false) Expect(err).NotTo(HaveOccurred()) }) @@ -60,7 +60,7 @@ var _ = Describe("Terraform Template", func() { }) Context("with valid input and a target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDir, outputDir, targetTemplateFile, ".spec.parameters[0]") + err := terraform(context.Background(), inputDir, outputDir, targetTemplateFile, ".spec.parameters[0]", false) Expect(err).NotTo(HaveOccurred()) }) @@ -83,7 +83,7 @@ var _ = Describe("Terraform Template", func() { }) Context("with valid input with required variable and a target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDirWithRequire, outputDir, targetTemplateFile, ".spec.parameters[0]") + err := terraform(context.Background(), inputDirWithRequire, outputDir, targetTemplateFile, ".spec.parameters[0]", false) Expect(err).NotTo(HaveOccurred()) }) @@ -98,7 +98,7 @@ var _ = Describe("Terraform Template", func() { Context("with a root directory specified", func() { BeforeEach(func() { - err := terraform(context.Background(), validInputRootDir, outputDir, "", "") + err := terraform(context.Background(), validInputRootDir, outputDir, "", "", false) Expect(err).NotTo(HaveOccurred()) }) @@ -118,8 +118,34 @@ var _ = Describe("Terraform Template", func() { Context("with an invalid input and no target template specified", func() { It("should return an error", func() { - err := terraform(context.Background(), "./fakes/terraform/invalid", outputDir, "", "") + err := terraform(context.Background(), "./fakes/terraform/invalid", outputDir, "", "", false) Expect(err).Should(HaveOccurred()) }) }) + + Context("with a root directory and oneOf flag specified", func() { + BeforeEach(func() { + err := terraform(context.Background(), validInputRootDir, outputDir, targetTemplateFile, ".spec.parameters[0]", true) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create properties files with properties merged and requirements updated", func() { + generatedInputData, err := os.ReadFile(fmt.Sprintf("%s/resources/input.yaml", outputDir)) + Expect(err).NotTo(HaveOccurred()) + generatedInputRequireData, err := os.ReadFile(fmt.Sprintf("%s/resources/input-require.yaml", outputDir)) + Expect(err).NotTo(HaveOccurred()) + expectedInputData, err := os.ReadFile(expectedPropertyFile) + Expect(err).NotTo(HaveOccurred()) + expectedInputRequireData, err := os.ReadFile(expectedPropertyFileWithRequire) + Expect(err).NotTo(HaveOccurred()) + + expectedTempalteData, err := os.ReadFile("./fakes/terraform/valid/output/full-template-oneof.yaml") + generatedTemplateData, err := os.ReadFile(fmt.Sprintf("%s/template.yaml", outputDir)) + + Expect(generatedInputData).To(MatchYAML(expectedInputData)) + Expect(generatedInputRequireData).To(MatchYAML(expectedInputRequireData)) + Expect(generatedTemplateData).To(MatchYAML(expectedTempalteData)) + + }) + }) }) diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go index d07cc8d..50ab8bd 100644 --- a/pkg/cmd/utils.go +++ b/pkg/cmd/utils.go @@ -3,12 +3,10 @@ package cmd import ( "context" "fmt" - "log" "os" "path/filepath" "strings" - "github.com/cnoe-io/cnoe-cli/pkg/models" "github.com/itchyny/gojq" yamlv3 "gopkg.in/yaml.v3" "sigs.k8s.io/yaml" @@ -16,21 +14,26 @@ import ( type finder func(file os.DirEntry, currentDepth uint32, base string) ([]string, error) -type NotFoundError struct { +type NotSupported struct { Err error } -func (n NotFoundError) Error() string { +func (n NotSupported) Error() string { return n.Error() } type insertAtInput struct { templatePath string jqPathExpression string - properties map[string]models.BackstageParamFields + fields map[string]any required []string } +type supportedFields struct { + Properties any `yaml:",omitempty"` + Dependencies any `yaml:",omitempty"` +} + func isDirectory(path string) bool { // Get file information info, err := os.Stat(path) @@ -38,11 +41,88 @@ func isDirectory(path string) bool { // Error occurred, path does not exist or cannot be accessed return false } - // Check if the path is a directory return info.Mode().IsDir() } +func checkAndCreateDir(path string) error { + _, err := os.Stat(path) + if os.IsNotExist(err) { + return os.MkdirAll(path, 0755) + } + return nil +} + +// return absolute path for given input, output, and template files, if output path does not exist, create it. +func prepDirectories(inputDir, outputDir, templateFile string, oneOf bool) (string, string, string, error) { + input, err := filepath.Abs(inputDir) + if err != nil { + return "", "", "", err + } + output, err := filepath.Abs(outputDir) + if err != nil { + return "", "", "", err + } + t, err := filepath.Abs(templateFile) + if err != nil { + return "", "", "", err + } + m := output + if oneOf { + m = filepath.Join(output, defDir) + } + err = checkAndCreateDir(m) + if err != nil { + return "", "", "", err + } + + return input, output, t, nil +} + +// Use the given template file, add dependencies and enum fields at the object specified by insertionPoint. +// Write the result to a file specified by outputFile. +func writeOneOf(ctx context.Context, outputFile, templatePath, insertionPoint string, resourceFiles []string) error { + input := insertAtInput{ + templatePath: templatePath, + jqPathExpression: insertionPoint, + } + t, err := oneOf(ctx, resourceFiles, input) + if err != nil { + return err + } + return writeOutput(t, outputFile) +} + +func oneOf(ctx context.Context, resourceFiles []string, input insertAtInput) (any, error) { + n := make([]string, len(resourceFiles)) + m := make([]map[string]string, len(resourceFiles)) + for i := range resourceFiles { + fileName := filepath.Base(resourceFiles[i]) + n[i] = strings.TrimSuffix(fileName, ".yaml") + m[i] = map[string]string{ + "$yaml": filepath.Join(defDir, fileName), + } + } + props := map[string]any{ + "resources": map[string]any{ + "type": "string", + "enum": n, + }, + } + deps := map[string]any{ + "resources": map[string][]map[string]string{ + "oneOf": m, + }, + } + fields := map[string]any{ + "properties": props, + "dependencies": deps, + } + input.fields = fields + + return insertAt(ctx, input) +} + func jsonFromObject(obj any) ([]byte, error) { b, err := yamlv3.Marshal(obj) if err != nil { @@ -62,14 +142,14 @@ func insertAt(ctx context.Context, input insertAtInput) (any, error) { if err != nil { return nil, err } - jqProp, err := jsonFromObject(input.properties) + jqProp, err := jsonFromObject(input.fields) if err != nil { return nil, err } var sb strings.Builder // update the properties field. merge (*) then assign (=) - sb.WriteString(fmt.Sprintf("%s.properties = %s.properties * %s", input.jqPathExpression, input.jqPathExpression, string(jqProp))) + sb.WriteString(fmt.Sprintf("%s = %s * %s", input.jqPathExpression, input.jqPathExpression, string(jqProp))) if len(input.required) > 0 { jqReq, err := jsonFromObject(input.required) if err != nil { @@ -81,12 +161,12 @@ func insertAt(ctx context.Context, input insertAtInput) (any, error) { } query, err := gojq.Parse(sb.String()) if err != nil { - log.Fatalln(err) + return nil, err } iter := query.RunWithContext(ctx, targetTemplate) v, _ := iter.Next() if err, ok := v.(error); ok { - log.Fatalln(err) + return nil, err } return v, nil } diff --git a/pkg/models/structs.go b/pkg/models/structs.go index 12ffc4b..fc76b8f 100644 --- a/pkg/models/structs.go +++ b/pkg/models/structs.go @@ -23,6 +23,7 @@ type Spec struct { } `yaml:"openAPIV3Schema"` } `json:"schema"` } `json:"versions"` + Scope string } type Definition struct { From 8d933b8a25829e9a0f631674c51b3f9aaef605cd Mon Sep 17 00:00:00 2001 From: Nima Kaviani Date: Sun, 27 Aug 2023 16:52:22 -0700 Subject: [PATCH 10/15] fix crd template compilation --- .gitignore | 1 + hack/build.sh | 1 + pkg/cmd/crd.go | 110 +++++++++--------------------------------- pkg/cmd/crd_test.go | 12 ++--- pkg/cmd/template.go | 17 ++++--- pkg/cmd/terraform.go | 10 ++-- pkg/cmd/utils.go | 17 +++---- pkg/models/structs.go | 29 ----------- 8 files changed, 55 insertions(+), 142 deletions(-) diff --git a/.gitignore b/.gitignore index 4e4c85f..8021fde 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ cnoe* go.sum +.vscode diff --git a/hack/build.sh b/hack/build.sh index 76cc21a..fd4449f 100755 --- a/hack/build.sh +++ b/hack/build.sh @@ -2,6 +2,7 @@ set -e -x -u +go mod tidy # go test ./... go fmt ./cmd/... ./pkg/... diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index 93b5249..7ffb995 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -16,7 +16,6 @@ import ( ) const ( - defDir = "resources" KindXRD = "CompositeResourceDefinition" KindCRD = "CustomResourceDefinition" ) @@ -56,54 +55,47 @@ type cmdOutput struct { func crd(cmd *cobra.Command, args []string) error { return Crd( cmd.Context(), - inputDir, outputDir, templatePath, insertionPoint, - verifiers, useOneOf, - templateName, templateTitle, templateDescription, + inputDir, outputDir, templatePath, insertionPoint, collapsed, + verifiers, templateName, templateTitle, templateDescription, ) } func Crd( - ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, - verifiers []string, useOneOf bool, - templateName, templateTitle, templateDescription string, + ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, collapsed bool, + verifiers []string, templateName, templateTitle, templateDescription string, ) error { - inDir, outDir, template, err := prepDirectories(inputDir, outputDir, templatePath, useOneOf) + inDir, expectedOutDir, template, err := prepDirectories(inputDir, outputDir, templatePath, collapsed) if err != nil { return err } + defs, err := getDefs(inDir, 0) if err != nil { return err } log.Printf("processing %d definitions", len(defs)) - schemaDir := outDir - if useOneOf { - schemaDir = filepath.Join(outDir, defDir) - output, err := writeSchema( - ctx, - schemaDir, - "", - "", - defs, - ) - if err != nil { - return err - } - templateFile := filepath.Join(outDir, "template.yaml") - return writeOneOf(ctx, templateFile, template, insertionPoint, output.Templates) - } - _, err = writeSchema( + output, err := writeSchema( ctx, - schemaDir, + expectedOutDir, insertionPoint, template, defs, + collapsed, ) if err != nil { return err } + if collapsed { + templateFile := filepath.Join(expectedOutDir, "../template.yaml") + input := insertAtInput{ + templatePath: template, + jqPathExpression: insertionPoint, + } + return writeOneOf(ctx, input, templateFile, output.Templates) + } + return nil } @@ -134,7 +126,7 @@ func findDefs(file os.DirEntry, currentDepth uint32, base string) ([]string, err return []string{f}, nil } -func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile string, defs []string) (cmdOutput, error) { +func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile string, defs []string, collapsed bool) (cmdOutput, error) { out := cmdOutput{ Templates: make([]string, 0), Resources: make([]string, 0), @@ -149,8 +141,10 @@ func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile st } return cmdOutput{}, err } + filename := filepath.Join(outputDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) - if templateFile != "" && insertionPoint != "" { + + if !collapsed { input := insertAtInput{ templatePath: templateFile, jqPathExpression: insertionPoint, @@ -168,6 +162,7 @@ func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile st } converted = t } + err = writeOutput(converted, filename) if err != nil { log.Printf("failed to write %s: %s \n", def, err.Error()) @@ -180,64 +175,6 @@ func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile st return out, nil } -func writeToTemplate( - templateFile string, outputPath string, identifiedResources []string, position int, - templateName, templateTitle, templateDescription string, -) error { - templateData, err := os.ReadFile(templateFile) - if err != nil { - return err - } - - var doc models.Template - err = yaml.Unmarshal(templateData, &doc) - if err != nil { - return err - } - - if templateName != "" { - doc.Metadata.Name = templateName - } - - if templateTitle != "" { - doc.Metadata.Title = templateTitle - } - - if templateDescription != "" { - doc.Metadata.Description = templateDescription - } - - dependencies := struct { - Resources struct { - OneOf []map[string]interface{} `yaml:"oneOf,omitempty"` - } `yaml:"resources,omitempty"` - }{} - - for _, r := range identifiedResources { - dependencies.Resources.OneOf = append(dependencies.Resources.OneOf, map[string]interface{}{ - "$yaml": fmt.Sprintf("resources/%s.yaml", strings.ToLower(r)), - }) - } - - if len(doc.Spec.Parameters) <= position { - return errors.New("not the right template or input format") - } - - doc.Spec.Parameters[position].Dependencies = dependencies - - outputData, err := yaml.Marshal(&doc) - if err != nil { - return err - } - - err = os.WriteFile(fmt.Sprintf("%s/template.yaml", outputPath), outputData, 0644) - if err != nil { - return err - } - - return nil -} - func convert(def string) (any, string, error) { data, err := os.ReadFile(def) if err != nil { @@ -257,6 +194,7 @@ func convert(def string) (any, string, error) { fmt.Errorf("%s is not a CRD or XRD", def), } } + var resourceName string if doc.Spec.ClaimNames != nil { resourceName = doc.Spec.ClaimNames.Kind diff --git a/pkg/cmd/crd_test.go b/pkg/cmd/crd_test.go index 7da2e9b..9fc30ec 100644 --- a/pkg/cmd/crd_test.go +++ b/pkg/cmd/crd_test.go @@ -46,8 +46,8 @@ var _ = Describe("Template CRDs", func() { Context("with valid input with oneof", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", - []string{}, true, templateName, templateTitle, templateDescription, + err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", true, + []string{}, templateName, templateTitle, templateDescription, ) Expect(err).NotTo(HaveOccurred()) }) @@ -81,8 +81,8 @@ var _ = Describe("Template CRDs", func() { Context("with valid input and specify template file and jq path", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", - []string{}, false, templateName, templateTitle, templateDescription, + err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", false, + []string{}, templateName, templateTitle, templateDescription, ) Expect(err).NotTo(HaveOccurred()) }) @@ -105,8 +105,8 @@ var _ = Describe("Template CRDs", func() { Context("with invalid input only", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), invalidInputDir, outputDir, "", "", - []string{}, false, templateName, templateTitle, templateDescription, + err := cmd.Crd(context.Background(), invalidInputDir, outputDir, "", "", false, + []string{}, templateName, templateTitle, templateDescription, ) Expect(err).NotTo(HaveOccurred()) }) diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index 04a44a9..d207cde 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -11,13 +11,17 @@ var templateCmd = &cobra.Command{ Short: "Generate Backstage templates", } +const ( + DefinitionsDir = "resources" +) + var ( depth uint32 insertionPoint string inputDir string outputDir string templatePath string - useOneOf bool + collapsed bool ) func init() { @@ -26,8 +30,8 @@ func init() { templateCmd.PersistentFlags().StringVarP(&outputDir, "outputDir", "o", "", "output directory for backstage templates to be stored in") templateCmd.PersistentFlags().StringVarP(&templatePath, "templatePath", "t", "", "path to the template to be augmented with backstage info") templateCmd.PersistentFlags().Uint32Var(&depth, "depth", 2, "depth from given directory to search for TF modules or CRDs") - templateCmd.PersistentFlags().StringVarP(&insertionPoint, "insertAt", "p", "", "jq path within the template to insert backstage info") - templateCmd.PersistentFlags().BoolVarP(&useOneOf, "oneTemplate", "u", false, "if set to true, items are rendered as drop down items in the specified template") + templateCmd.PersistentFlags().StringVarP(&insertionPoint, "insertAt", "p", ".spec.parameters[0]", "jq path within the template to insert backstage info") + templateCmd.PersistentFlags().BoolVarP(&collapsed, "collapse", "c", false, "if set to true, items are rendered and collapsed as drop down items in a single specified template") templateCmd.MarkFlagRequired("inputDir") templateCmd.MarkFlagRequired("outputDir") @@ -38,12 +42,9 @@ func templatePreRunE(cmd *cobra.Command, args []string) error { return errors.New("inputDir must be a directory") } - if useOneOf && templatePath == "" && insertionPoint == "" { - return errors.New("templatePath and inertAt flags must be specified when using the oneTemplate flag") + if collapsed && templatePath == "" { + return errors.New("templatePath flag must be specified when using the `collapse` flag (optionally you can use `insertAt` as well.)") } - if insertionPoint != "" && templatePath == "" { - return errors.New("templatePath flag must be specified") - } return nil } diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index 6f371f0..793941f 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -32,7 +32,7 @@ func init() { } func tfE(cmd *cobra.Command, args []string) error { - return terraform(cmd.Context(), inputDir, outputDir, templatePath, insertionPoint, useOneOf) + return terraform(cmd.Context(), inputDir, outputDir, templatePath, insertionPoint, collapsed) } func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, useOneOf bool) error { @@ -66,7 +66,7 @@ func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertion } } if useOneOf { - p := filepath.Join(outDir, defDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) + p := filepath.Join(outDir, DefinitionsDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) log.Printf("writing to %s", p) err := handleOutput(ctx, p, "", "", params, required) if err != nil { @@ -87,7 +87,11 @@ func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertion for i := range mods { resourceFileNames[i] = fmt.Sprintf("%s.yaml", mods[i]) } - return writeOneOf(ctx, templateFile, template, insertionPoint, resourceFileNames) + input := insertAtInput{ + templatePath: template, + jqPathExpression: insertionPoint, + } + return writeOneOf(ctx, input, templateFile, resourceFileNames) } return nil } diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go index 50ab8bd..e7cda67 100644 --- a/pkg/cmd/utils.go +++ b/pkg/cmd/utils.go @@ -67,25 +67,22 @@ func prepDirectories(inputDir, outputDir, templateFile string, oneOf bool) (stri if err != nil { return "", "", "", err } - m := output + expectedOutput := output if oneOf { - m = filepath.Join(output, defDir) + expectedOutput = filepath.Join(output, DefinitionsDir) } - err = checkAndCreateDir(m) + err = checkAndCreateDir(expectedOutput) if err != nil { return "", "", "", err } - return input, output, t, nil + return input, expectedOutput, t, nil } // Use the given template file, add dependencies and enum fields at the object specified by insertionPoint. // Write the result to a file specified by outputFile. -func writeOneOf(ctx context.Context, outputFile, templatePath, insertionPoint string, resourceFiles []string) error { - input := insertAtInput{ - templatePath: templatePath, - jqPathExpression: insertionPoint, - } +func writeOneOf(ctx context.Context, input insertAtInput, outputFile string, resourceFiles []string) error { + t, err := oneOf(ctx, resourceFiles, input) if err != nil { return err @@ -100,7 +97,7 @@ func oneOf(ctx context.Context, resourceFiles []string, input insertAtInput) (an fileName := filepath.Base(resourceFiles[i]) n[i] = strings.TrimSuffix(fileName, ".yaml") m[i] = map[string]string{ - "$yaml": filepath.Join(defDir, fileName), + "$yaml": filepath.Join(DefinitionsDir, fileName), } } props := map[string]any{ diff --git a/pkg/models/structs.go b/pkg/models/structs.go index fc76b8f..1c3e15b 100644 --- a/pkg/models/structs.go +++ b/pkg/models/structs.go @@ -52,35 +52,6 @@ type Wrapper struct { Properties Props `json:"properties"` } -type Template struct { - ApiVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind"` - Metadata struct { - Name string `yaml:"name"` - Title string `yaml:"title"` - Description string `yaml:"description"` - } `yaml:"metadata"` - Spec struct { - Owner string `yaml:"owner"` - Type string `yaml:"type"` - Parameters []struct { - Properties map[string]interface{} `yaml:"properties"` - Dependencies struct { - Resources struct { - OneOf []map[string]interface{} `yaml:"oneOf,omitempty"` - } `yaml:"resources,omitempty"` - } `yaml:"dependencies,omitempty"` - } `yaml:"parameters"` - - Steps []struct { - Id string `yaml:"id"` - Name string `yaml:"name"` - Action string `yaml:"action"` - Input map[string]interface{} `yaml:"input"` - } `yaml:"steps"` - } `yaml:"spec"` -} - type BackstageParamFields struct { Title string `yaml:",omitempty"` Type string From a5b1552559c96c9e9342599a9f6a239f5fca2b01 Mon Sep 17 00:00:00 2001 From: Nima Kaviani Date: Sun, 27 Aug 2023 18:09:25 -0700 Subject: [PATCH 11/15] fix terraform templating - add the raw flag for when only OpenAPI spec is required - remove implicit assumption on the use of flags - simplify the code --- pkg/cmd/crd.go | 17 ++++++++++----- pkg/cmd/crd_test.go | 6 ++--- pkg/cmd/template.go | 8 ++++++- pkg/cmd/terraform.go | 46 ++++++++++++++++++++------------------- pkg/cmd/terraform_test.go | 12 +++++----- 5 files changed, 51 insertions(+), 38 deletions(-) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index 7ffb995..bad5af3 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -55,16 +55,20 @@ type cmdOutput struct { func crd(cmd *cobra.Command, args []string) error { return Crd( cmd.Context(), - inputDir, outputDir, templatePath, insertionPoint, collapsed, + inputDir, outputDir, templatePath, insertionPoint, collapsed, raw, verifiers, templateName, templateTitle, templateDescription, ) } func Crd( - ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, collapsed bool, + ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, collapsed, raw bool, verifiers []string, templateName, templateTitle, templateDescription string, ) error { - inDir, expectedOutDir, template, err := prepDirectories(inputDir, outputDir, templatePath, collapsed) + inDir, expectedOutDir, template, err := prepDirectories( + inputDir, + outputDir, + templatePath, + collapsed && !raw /* only generate nesting if needs to be collapsed and not be raw*/) if err != nil { return err } @@ -82,12 +86,13 @@ func Crd( template, defs, collapsed, + raw, ) if err != nil { return err } - if collapsed { + if collapsed && !raw { templateFile := filepath.Join(expectedOutDir, "../template.yaml") input := insertAtInput{ templatePath: template, @@ -126,7 +131,7 @@ func findDefs(file os.DirEntry, currentDepth uint32, base string) ([]string, err return []string{f}, nil } -func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile string, defs []string, collapsed bool) (cmdOutput, error) { +func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile string, defs []string, collapsed, raw bool) (cmdOutput, error) { out := cmdOutput{ Templates: make([]string, 0), Resources: make([]string, 0), @@ -144,7 +149,7 @@ func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile st filename := filepath.Join(outputDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) - if !collapsed { + if !collapsed && !raw { input := insertAtInput{ templatePath: templateFile, jqPathExpression: insertionPoint, diff --git a/pkg/cmd/crd_test.go b/pkg/cmd/crd_test.go index 9fc30ec..1fe8ff5 100644 --- a/pkg/cmd/crd_test.go +++ b/pkg/cmd/crd_test.go @@ -46,7 +46,7 @@ var _ = Describe("Template CRDs", func() { Context("with valid input with oneof", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", true, + err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", true, false, []string{}, templateName, templateTitle, templateDescription, ) Expect(err).NotTo(HaveOccurred()) @@ -81,7 +81,7 @@ var _ = Describe("Template CRDs", func() { Context("with valid input and specify template file and jq path", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", false, + err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", false, false, []string{}, templateName, templateTitle, templateDescription, ) Expect(err).NotTo(HaveOccurred()) @@ -105,7 +105,7 @@ var _ = Describe("Template CRDs", func() { Context("with invalid input only", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), invalidInputDir, outputDir, "", "", false, + err := cmd.Crd(context.Background(), invalidInputDir, outputDir, "", "", false, false, []string{}, templateName, templateTitle, templateDescription, ) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index d207cde..043af21 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -22,6 +22,7 @@ var ( outputDir string templatePath string collapsed bool + raw bool ) func init() { @@ -32,6 +33,7 @@ func init() { templateCmd.PersistentFlags().Uint32Var(&depth, "depth", 2, "depth from given directory to search for TF modules or CRDs") templateCmd.PersistentFlags().StringVarP(&insertionPoint, "insertAt", "p", ".spec.parameters[0]", "jq path within the template to insert backstage info") templateCmd.PersistentFlags().BoolVarP(&collapsed, "collapse", "c", false, "if set to true, items are rendered and collapsed as drop down items in a single specified template") + templateCmd.PersistentFlags().BoolVarP(&raw, "raw", "", false, "prints the raw open API output without putting it into a template (ignoring `templatePath` and `insertAt`)") templateCmd.MarkFlagRequired("inputDir") templateCmd.MarkFlagRequired("outputDir") @@ -42,9 +44,13 @@ func templatePreRunE(cmd *cobra.Command, args []string) error { return errors.New("inputDir must be a directory") } - if collapsed && templatePath == "" { + if collapsed && templatePath == "" && !raw { return errors.New("templatePath flag must be specified when using the `collapse` flag (optionally you can use `insertAt` as well.)") } + if templatePath == "" && !raw { + return errors.New("you either need to use the `raw` flag to generate raw OpenAPI files or define a `templatePath` for the tool to populate.") + } + return nil } diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index 793941f..c51b206 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -32,11 +32,19 @@ func init() { } func tfE(cmd *cobra.Command, args []string) error { - return terraform(cmd.Context(), inputDir, outputDir, templatePath, insertionPoint, collapsed) + return terraform(cmd.Context(), inputDir, outputDir, templatePath, insertionPoint, collapsed, raw) } -func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, useOneOf bool) error { - inDir, outDir, template, err := prepDirectories(inputDir, outputDir, templatePath, useOneOf) +func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, collapsed, raw bool) error { + inDir, expectedOutDir, template, err := prepDirectories( + inputDir, + outputDir, + templatePath, + collapsed && !raw /* only generate nesting if needs to be collapsed and not be raw*/) + if err != nil { + return err + } + mods, err := getModules(inDir, 0) if err != nil { return err @@ -53,9 +61,10 @@ func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertion return diag.Err() } if len(mod.Variables) == 0 { - log.Printf(fmt.Sprintf("module %s does not have variables", path)) + log.Printf("module %s does not have variables", path) continue } + params := make(map[string]models.BackstageParamFields) required := make([]string, 0) log.Printf("converting %d variables", len(mod.Variables)) @@ -65,24 +74,17 @@ func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertion required = append(required, j) } } - if useOneOf { - p := filepath.Join(outDir, DefinitionsDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) - log.Printf("writing to %s", p) - err := handleOutput(ctx, p, "", "", params, required) - if err != nil { - return err - } - } else { - p := filepath.Join(outDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) - log.Printf("writing to %s", p) - err := handleOutput(ctx, p, template, insertionPoint, params, required) - if err != nil { - return err - } + + p := filepath.Join(expectedOutDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) + log.Printf("writing to %s", p) + err = handleOutput(ctx, p, template, insertionPoint, params, required, collapsed, raw) + if err != nil { + return err } } - if useOneOf { - templateFile := filepath.Join(outDir, "template.yaml") + + if collapsed && !raw { + templateFile := filepath.Join(expectedOutDir, "../template.yaml") resourceFileNames := make([]string, len(mods)) for i := range mods { resourceFileNames[i] = fmt.Sprintf("%s.yaml", mods[i]) @@ -96,12 +98,12 @@ func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertion return nil } -func handleOutput(ctx context.Context, outputFile, templateFile, insertionPoint string, properties map[string]models.BackstageParamFields, required []string) error { +func handleOutput(ctx context.Context, outputFile, templateFile, insertionPoint string, properties map[string]models.BackstageParamFields, required []string, collapsed, raw bool) error { props := make(map[string]any, len(properties)) for k := range properties { props[k] = properties[k] } - if templateFile != "" && insertionPoint != "" { + if !raw && !collapsed { input := insertAtInput{ templatePath: templateFile, jqPathExpression: insertionPoint, diff --git a/pkg/cmd/terraform_test.go b/pkg/cmd/terraform_test.go index 29f97af..cafbe45 100644 --- a/pkg/cmd/terraform_test.go +++ b/pkg/cmd/terraform_test.go @@ -45,7 +45,7 @@ var _ = Describe("Terraform Template", func() { Context("with valid input and no target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDir, outputDir, "", "", false) + err := terraform(context.Background(), inputDir, outputDir, "", "", false, true) Expect(err).NotTo(HaveOccurred()) }) @@ -60,7 +60,7 @@ var _ = Describe("Terraform Template", func() { }) Context("with valid input and a target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDir, outputDir, targetTemplateFile, ".spec.parameters[0]", false) + err := terraform(context.Background(), inputDir, outputDir, targetTemplateFile, ".spec.parameters[0]", false, false) Expect(err).NotTo(HaveOccurred()) }) @@ -83,7 +83,7 @@ var _ = Describe("Terraform Template", func() { }) Context("with valid input with required variable and a target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDirWithRequire, outputDir, targetTemplateFile, ".spec.parameters[0]", false) + err := terraform(context.Background(), inputDirWithRequire, outputDir, targetTemplateFile, ".spec.parameters[0]", false, false) Expect(err).NotTo(HaveOccurred()) }) @@ -98,7 +98,7 @@ var _ = Describe("Terraform Template", func() { Context("with a root directory specified", func() { BeforeEach(func() { - err := terraform(context.Background(), validInputRootDir, outputDir, "", "", false) + err := terraform(context.Background(), validInputRootDir, outputDir, "", "", false, true) Expect(err).NotTo(HaveOccurred()) }) @@ -118,14 +118,14 @@ var _ = Describe("Terraform Template", func() { Context("with an invalid input and no target template specified", func() { It("should return an error", func() { - err := terraform(context.Background(), "./fakes/terraform/invalid", outputDir, "", "", false) + err := terraform(context.Background(), "./fakes/terraform/invalid", outputDir, "", "", false, false) Expect(err).Should(HaveOccurred()) }) }) Context("with a root directory and oneOf flag specified", func() { BeforeEach(func() { - err := terraform(context.Background(), validInputRootDir, outputDir, targetTemplateFile, ".spec.parameters[0]", true) + err := terraform(context.Background(), validInputRootDir, outputDir, targetTemplateFile, ".spec.parameters[0]", true, false) Expect(err).NotTo(HaveOccurred()) }) From d5ab2b9042b2717bef46e4502cce373155e653f5 Mon Sep 17 00:00:00 2001 From: Nima Kaviani Date: Sun, 27 Aug 2023 23:20:18 -0700 Subject: [PATCH 12/15] combine common parts of the code across terraform and crds --- pkg/cmd/crd.go | 109 +++++++++++++++------------------ pkg/cmd/crd_test.go | 12 ++-- pkg/cmd/template.go | 84 +++++++++++++++++++++++++- pkg/cmd/terraform.go | 124 ++++++++++++++++++++++++-------------- pkg/cmd/terraform_test.go | 15 ++--- 5 files changed, 223 insertions(+), 121 deletions(-) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index bad5af3..2b2a0e7 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -47,82 +47,67 @@ var ( } ) -type cmdOutput struct { - Templates []string - Resources []string -} - func crd(cmd *cobra.Command, args []string) error { - return Crd( + return Process( cmd.Context(), - inputDir, outputDir, templatePath, insertionPoint, collapsed, raw, - verifiers, templateName, templateTitle, templateDescription, + NewCRDModule( + inputDir, outputDir, templatePath, insertionPoint, collapsed, raw, + verifiers, templateName, templateTitle, templateDescription, + ), ) } -func Crd( - ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, collapsed, raw bool, - verifiers []string, templateName, templateTitle, templateDescription string, -) error { - inDir, expectedOutDir, template, err := prepDirectories( - inputDir, - outputDir, - templatePath, - collapsed && !raw /* only generate nesting if needs to be collapsed and not be raw*/) - if err != nil { - return err - } - - defs, err := getDefs(inDir, 0) - if err != nil { - return err - } - log.Printf("processing %d definitions", len(defs)) - - output, err := writeSchema( - ctx, - expectedOutDir, - insertionPoint, - template, - defs, - collapsed, - raw, - ) - if err != nil { - return err - } +type CRDModule struct { + EntityConfig + verifiers []string + templateName string + templateTitle string + templateDescription string +} - if collapsed && !raw { - templateFile := filepath.Join(expectedOutDir, "../template.yaml") - input := insertAtInput{ - templatePath: template, - jqPathExpression: insertionPoint, - } - return writeOneOf(ctx, input, templateFile, output.Templates) +func NewCRDModule( + inputDir, outputDir, templatePath, insertionPoint string, collapsed, raw bool, + verifiers []string, templateName, templateTitle, templateDescription string, +) Entity { + return &CRDModule{ + EntityConfig: EntityConfig{ + InputDir: inputDir, + OutputDir: outputDir, + TemplateFile: templatePath, + InsertionPoint: insertionPoint, + Collapsed: collapsed, + Raw: raw, + }, + verifiers: verifiers, + templateName: templateName, + templateTitle: templateTitle, + templateDescription: templateDescription, } +} - return nil +func (c *CRDModule) Config() EntityConfig { + return c.EntityConfig } -func getDefs(inputDir string, currentDepth uint32) ([]string, error) { +func (c *CRDModule) GetDefinitions(inputDir string, currentDepth uint32) ([]string, error) { if currentDepth > depth { return nil, nil } - out, err := getRelevantFiles(inputDir, currentDepth, findDefs) + out, err := getRelevantFiles(inputDir, currentDepth, c.findDefs) if err != nil { return nil, err } return out, nil } -func findDefs(file os.DirEntry, currentDepth uint32, base string) ([]string, error) { +func (c *CRDModule) findDefs(file os.DirEntry, currentDepth uint32, base string) ([]string, error) { f := filepath.Join(base, file.Name()) stat, err := os.Stat(f) if err != nil { return nil, err } if stat.IsDir() { - df, err := getDefs(f, currentDepth+1) + df, err := c.GetDefinitions(f, currentDepth+1) if err != nil { return nil, err } @@ -131,28 +116,28 @@ func findDefs(file os.DirEntry, currentDepth uint32, base string) ([]string, err return []string{f}, nil } -func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile string, defs []string, collapsed, raw bool) (cmdOutput, error) { - out := cmdOutput{ +func (c *CRDModule) HandleEntries(ctx context.Context, cc EntryConfig) (ProcessOutput, error) { + out := ProcessOutput{ Templates: make([]string, 0), Resources: make([]string, 0), } - for _, def := range defs { + for _, def := range cc.Definitions { converted, resourceName, err := convert(def) if err != nil { var e NotSupported if errors.As(err, &e) { continue } - return cmdOutput{}, err + return ProcessOutput{}, err } - filename := filepath.Join(outputDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) + filename := filepath.Join(cc.ExpectedOutDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) - if !collapsed && !raw { + if !cc.Collapsed && !cc.Raw { input := insertAtInput{ - templatePath: templateFile, - jqPathExpression: insertionPoint, + templatePath: cc.TemplateFile, + jqPathExpression: cc.InsertionPoint, } props := converted.(map[string]any) if v, reqOk := props["required"]; reqOk { @@ -163,7 +148,7 @@ func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile st input.fields = props t, err := insertAt(ctx, input) if err != nil { - return cmdOutput{}, err + return ProcessOutput{}, err } converted = t } @@ -171,7 +156,7 @@ func writeSchema(ctx context.Context, outputDir, insertionPoint, templateFile st err = writeOutput(converted, filename) if err != nil { log.Printf("failed to write %s: %s \n", def, err.Error()) - return cmdOutput{}, err + return ProcessOutput{}, err } out.Templates = append(out.Templates, filename) out.Resources = append(out.Resources, resourceName) @@ -215,7 +200,7 @@ func convert(def string) (any, string, error) { } else { value, err = ConvertMap(v) if err != nil { - return cmdOutput{}, "", err + return ProcessOutput{}, "", err } } diff --git a/pkg/cmd/crd_test.go b/pkg/cmd/crd_test.go index 1fe8ff5..d94609d 100644 --- a/pkg/cmd/crd_test.go +++ b/pkg/cmd/crd_test.go @@ -46,9 +46,9 @@ var _ = Describe("Template CRDs", func() { Context("with valid input with oneof", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", true, false, + err := cmd.Process(context.Background(), cmd.NewCRDModule(inputDir, outputDir, templateFile, ".spec.parameters[0]", true, false, []string{}, templateName, templateTitle, templateDescription, - ) + )) Expect(err).NotTo(HaveOccurred()) }) @@ -81,9 +81,9 @@ var _ = Describe("Template CRDs", func() { Context("with valid input and specify template file and jq path", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), inputDir, outputDir, templateFile, ".spec.parameters[0]", false, false, + err := cmd.Process(context.Background(), cmd.NewCRDModule(inputDir, outputDir, templateFile, ".spec.parameters[0]", false, false, []string{}, templateName, templateTitle, templateDescription, - ) + )) Expect(err).NotTo(HaveOccurred()) }) @@ -105,9 +105,9 @@ var _ = Describe("Template CRDs", func() { Context("with invalid input only", func() { BeforeEach(func() { - err := cmd.Crd(context.Background(), invalidInputDir, outputDir, "", "", false, false, + err := cmd.Process(context.Background(), cmd.NewCRDModule(invalidInputDir, outputDir, "", "", false, false, []string{}, templateName, templateTitle, templateDescription, - ) + )) Expect(err).NotTo(HaveOccurred()) }) diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index 043af21..d52a7fc 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -1,8 +1,12 @@ package cmd import ( + "context" "errors" + "log" + "path/filepath" + "github.com/cnoe-io/cnoe-cli/pkg/models" "github.com/spf13/cobra" ) @@ -32,8 +36,8 @@ func init() { templateCmd.PersistentFlags().StringVarP(&templatePath, "templatePath", "t", "", "path to the template to be augmented with backstage info") templateCmd.PersistentFlags().Uint32Var(&depth, "depth", 2, "depth from given directory to search for TF modules or CRDs") templateCmd.PersistentFlags().StringVarP(&insertionPoint, "insertAt", "p", ".spec.parameters[0]", "jq path within the template to insert backstage info") - templateCmd.PersistentFlags().BoolVarP(&collapsed, "collapse", "c", false, "if set to true, items are rendered and collapsed as drop down items in a single specified template") - templateCmd.PersistentFlags().BoolVarP(&raw, "raw", "", false, "prints the raw open API output without putting it into a template (ignoring `templatePath` and `insertAt`)") + templateCmd.PersistentFlags().BoolVarP(&collapsed, "colllapse", "c", false, "if set to true, items are rendered and collapsed as drop down items in a single specified template") + templateCmd.PersistentFlags().BoolVarP(&raw, "raww", "", false, "prints the raw open API output without putting it into a template (ignoring `templatePath` and `insertAt`)") templateCmd.MarkFlagRequired("inputDir") templateCmd.MarkFlagRequired("outputDir") @@ -54,3 +58,79 @@ func templatePreRunE(cmd *cobra.Command, args []string) error { return nil } + +type EntityConfig struct { + InputDir string + OutputDir string + TemplateFile string + OutputFile string + InsertionPoint string + Defenitions []string + Properties map[string]models.BackstageParamFields + Required []string + Collapsed bool + Raw bool +} + +type EntryConfig struct { + Definitions []string + ExpectedOutDir string + TemplateFile string + InsertionPoint string + Collapsed bool + Raw bool +} + +type ProcessOutput struct { + Templates []string + Resources []string +} + +type Entity interface { + GetDefinitions(string, uint32) ([]string, error) + HandleEntries(context.Context, EntryConfig) (ProcessOutput, error) + Config() EntityConfig +} + +func Process(ctx context.Context, p Entity) error { + c := p.Config() + + expectedInDir, expectedOutDir, expectedTemplateFile, err := prepDirectories( + c.InputDir, + c.OutputDir, + c.TemplateFile, + c.Collapsed && !c.Raw, /* only generate nesting if needs to be collapsed and not be raw*/ + ) + if err != nil { + return err + } + + defs, err := p.GetDefinitions(expectedInDir, 0) + if err != nil { + return err + } + log.Printf("processing %d definitions", len(defs)) + + output, err := p.HandleEntries(ctx, EntryConfig{ + Definitions: defs, + ExpectedOutDir: expectedOutDir, + TemplateFile: expectedTemplateFile, + InsertionPoint: c.InsertionPoint, + Collapsed: c.Collapsed, + Raw: c.Raw, + }) + if err != nil { + return err + } + + if c.Collapsed && !c.Raw { + generatedTemplateFile := filepath.Join(expectedOutDir, "../template.yaml") + input := insertAtInput{ + templatePath: expectedTemplateFile, + jqPathExpression: c.InsertionPoint, + } + return writeOneOf(ctx, input, generatedTemplateFile, output.Templates) + } + + return nil +} diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index c51b206..f9048cd 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -32,70 +32,106 @@ func init() { } func tfE(cmd *cobra.Command, args []string) error { - return terraform(cmd.Context(), inputDir, outputDir, templatePath, insertionPoint, collapsed, raw) + return Process(cmd.Context(), NewTerraformModule(inputDir, outputDir, templatePath, insertionPoint, collapsed, raw)) } -func terraform(ctx context.Context, inputDir, outputDir, templatePath, insertionPoint string, collapsed, raw bool) error { - inDir, expectedOutDir, template, err := prepDirectories( - inputDir, - outputDir, - templatePath, - collapsed && !raw /* only generate nesting if needs to be collapsed and not be raw*/) - if err != nil { - return err - } +type TerraformModule struct { + EntityConfig +} - mods, err := getModules(inDir, 0) - if err != nil { - return err +func NewTerraformModule(inputDir, outputDir, templatePath, insertionPoint string, collapsed, raw bool) Entity { + + return &TerraformModule{ + EntityConfig: EntityConfig{ + InputDir: inputDir, + OutputDir: outputDir, + TemplateFile: templatePath, + InsertionPoint: insertionPoint, + Collapsed: collapsed, + Raw: raw, + }, } - if len(mods) == 0 { - return fmt.Errorf("could not find any TF modules in given directorr: %s", inDir) +} + +func (t *TerraformModule) Config() EntityConfig { + return t.EntityConfig +} + +func (t *TerraformModule) HandleEntries(ctx context.Context, c EntryConfig) (ProcessOutput, error) { + out := ProcessOutput{ + Templates: make([]string, 0), + Resources: make([]string, 0), } - log.Printf("processing %d modules", len(mods)) - for i := range mods { - path := mods[i] + + for i := range c.Definitions { + path := c.Definitions[i] log.Printf("processing module at %s", path) mod, diag := tfconfig.LoadModule(path) if diag.HasErrors() { - return diag.Err() + return out, diag.Err() } + if len(mod.Variables) == 0 { log.Printf("module %s does not have variables", path) continue } - params := make(map[string]models.BackstageParamFields) - required := make([]string, 0) - log.Printf("converting %d variables", len(mod.Variables)) - for j := range mod.Variables { - params[j] = convertVariable(*mod.Variables[j]) - if mod.Variables[j].Required { - required = append(required, j) - } - } + params, required := prepareParamsAndRequired(mod) - p := filepath.Join(expectedOutDir, fmt.Sprintf("%s.yaml", filepath.Base(path))) - log.Printf("writing to %s", p) - err = handleOutput(ctx, p, template, insertionPoint, params, required, collapsed, raw) + fileName := fmt.Sprintf("%s.yaml", filepath.Base(path)) + outputFile := filepath.Join(c.ExpectedOutDir, fileName) + err := handleModuleOutput(ctx, outputFile, c.TemplateFile, c.InsertionPoint, params, required, c.Collapsed, c.Raw) if err != nil { - return err + return ProcessOutput{}, err } + + out.Templates = append(out.Templates, fileName) } - if collapsed && !raw { - templateFile := filepath.Join(expectedOutDir, "../template.yaml") - resourceFileNames := make([]string, len(mods)) - for i := range mods { - resourceFileNames[i] = fmt.Sprintf("%s.yaml", mods[i]) + return out, nil +} + +func prepareParamsAndRequired(mod *tfconfig.Module) (map[string]models.BackstageParamFields, []string) { + params := make(map[string]models.BackstageParamFields) + required := make([]string, 0) + for j := range mod.Variables { + params[j] = convertVariable(*mod.Variables[j]) + if mod.Variables[j].Required { + required = append(required, j) } + } + return params, required +} + +func handleModuleOutput(ctx context.Context, outputFile, templateFile, insertionPoint string, properties map[string]models.BackstageParamFields, required []string, collapsed, raw bool) error { + props := make(map[string]interface{}, len(properties)) + for k := range properties { + props[k] = properties[k] + } + + if !raw && !collapsed { input := insertAtInput{ - templatePath: template, + templatePath: templateFile, jqPathExpression: insertionPoint, + fields: map[string]interface{}{ + "properties": props, + }, + } + if len(required) > 0 { + input.required = required } - return writeOneOf(ctx, input, templateFile, resourceFileNames) + t, err := insertAt(ctx, input) + if err != nil { + return fmt.Errorf("failed to insert to given template: %s", err) + } + return writeOutput(t, outputFile) } - return nil + + t := map[string]interface{}{ + "properties": props, + "required": required, + } + return writeOutput(t, outputFile) } func handleOutput(ctx context.Context, outputFile, templateFile, insertionPoint string, properties map[string]models.BackstageParamFields, required []string, collapsed, raw bool) error { @@ -127,28 +163,28 @@ func handleOutput(ctx context.Context, outputFile, templateFile, insertionPoint return writeOutput(t, outputFile) } -func getModules(inputDir string, currentDepth uint32) ([]string, error) { +func (t *TerraformModule) GetDefinitions(inputDir string, currentDepth uint32) ([]string, error) { if currentDepth > depth { return nil, nil } if tfconfig.IsModuleDir(inputDir) { return []string{inputDir}, nil } - out, err := getRelevantFiles(inputDir, currentDepth, findModule) + out, err := getRelevantFiles(inputDir, currentDepth, t.findModule) if err != nil { return nil, err } return out, nil } -func findModule(file os.DirEntry, currentDepth uint32, base string) ([]string, error) { +func (t *TerraformModule) findModule(file os.DirEntry, currentDepth uint32, base string) ([]string, error) { f := filepath.Join(base, file.Name()) stat, err := os.Stat(f) if err != nil { return nil, err } if stat.IsDir() { - mods, err := getModules(f, currentDepth+1) + mods, err := t.GetDefinitions(f, currentDepth+1) if err != nil { return nil, err } diff --git a/pkg/cmd/terraform_test.go b/pkg/cmd/terraform_test.go index cafbe45..ed66cef 100644 --- a/pkg/cmd/terraform_test.go +++ b/pkg/cmd/terraform_test.go @@ -1,4 +1,4 @@ -package cmd +package cmd_test import ( "context" @@ -7,6 +7,7 @@ import ( "os" "path/filepath" + "github.com/cnoe-io/cnoe-cli/pkg/cmd" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -45,7 +46,7 @@ var _ = Describe("Terraform Template", func() { Context("with valid input and no target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDir, outputDir, "", "", false, true) + err := cmd.Process(context.Background(), cmd.NewTerraformModule(inputDir, outputDir, "", "", false, true)) Expect(err).NotTo(HaveOccurred()) }) @@ -60,7 +61,7 @@ var _ = Describe("Terraform Template", func() { }) Context("with valid input and a target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDir, outputDir, targetTemplateFile, ".spec.parameters[0]", false, false) + err := cmd.Process(context.Background(), cmd.NewTerraformModule(inputDir, outputDir, targetTemplateFile, ".spec.parameters[0]", false, false)) Expect(err).NotTo(HaveOccurred()) }) @@ -83,7 +84,7 @@ var _ = Describe("Terraform Template", func() { }) Context("with valid input with required variable and a target template specified", func() { BeforeEach(func() { - err := terraform(context.Background(), inputDirWithRequire, outputDir, targetTemplateFile, ".spec.parameters[0]", false, false) + err := cmd.Process(context.Background(), cmd.NewTerraformModule(inputDirWithRequire, outputDir, targetTemplateFile, ".spec.parameters[0]", false, false)) Expect(err).NotTo(HaveOccurred()) }) @@ -98,7 +99,7 @@ var _ = Describe("Terraform Template", func() { Context("with a root directory specified", func() { BeforeEach(func() { - err := terraform(context.Background(), validInputRootDir, outputDir, "", "", false, true) + err := cmd.Process(context.Background(), cmd.NewTerraformModule(validInputRootDir, outputDir, "", "", false, true)) Expect(err).NotTo(HaveOccurred()) }) @@ -118,14 +119,14 @@ var _ = Describe("Terraform Template", func() { Context("with an invalid input and no target template specified", func() { It("should return an error", func() { - err := terraform(context.Background(), "./fakes/terraform/invalid", outputDir, "", "", false, false) + err := cmd.Process(context.Background(), cmd.NewTerraformModule("./fakes/terraform/invalid", outputDir, "", "", false, false)) Expect(err).Should(HaveOccurred()) }) }) Context("with a root directory and oneOf flag specified", func() { BeforeEach(func() { - err := terraform(context.Background(), validInputRootDir, outputDir, targetTemplateFile, ".spec.parameters[0]", true, false) + err := cmd.Process(context.Background(), cmd.NewTerraformModule(validInputRootDir, outputDir, targetTemplateFile, ".spec.parameters[0]", true, false)) Expect(err).NotTo(HaveOccurred()) }) From 7d417170b5566bcee0201602d0582e73c795c4b4 Mon Sep 17 00:00:00 2001 From: Nima Kaviani Date: Mon, 28 Aug 2023 19:00:13 -0700 Subject: [PATCH 13/15] more cleanup --- pkg/cmd/crd.go | 8 ++-- pkg/cmd/template.go | 9 ----- pkg/cmd/terraform.go | 85 +++++---------------------------------- pkg/cmd/terraform_test.go | 7 ++++ 4 files changed, 21 insertions(+), 88 deletions(-) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index 2b2a0e7..ebaee20 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -134,10 +134,10 @@ func (c *CRDModule) HandleEntries(ctx context.Context, cc EntryConfig) (ProcessO filename := filepath.Join(cc.ExpectedOutDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) - if !cc.Collapsed && !cc.Raw { + if !c.Collapsed && !c.Raw { input := insertAtInput{ templatePath: cc.TemplateFile, - jqPathExpression: cc.InsertionPoint, + jqPathExpression: c.InsertionPoint, } props := converted.(map[string]any) if v, reqOk := props["required"]; reqOk { @@ -277,7 +277,7 @@ func ConvertMap(originalData interface{}) (map[string]interface{}, error) { var err error convertedMap[key], err = ConvertMap(v) if err != nil { - return nil, errors.New(fmt.Sprintf("failed to convert for key %s", key)) + return nil, fmt.Errorf("failed to convert for key %s", key) } case int: convertedMap[key] = int64(v) @@ -290,7 +290,7 @@ func ConvertMap(originalData interface{}) (map[string]interface{}, error) { case map[interface{}]interface{}: ivec, err := ConvertMap(ive) if err != nil { - return nil, errors.New(fmt.Sprintf("failed to convert for key %s", key)) + return nil, fmt.Errorf("failed to convert for key %s", key) } dv[i] = ivec case int: diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index d52a7fc..f769314 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -6,7 +6,6 @@ import ( "log" "path/filepath" - "github.com/cnoe-io/cnoe-cli/pkg/models" "github.com/spf13/cobra" ) @@ -66,8 +65,6 @@ type EntityConfig struct { OutputFile string InsertionPoint string Defenitions []string - Properties map[string]models.BackstageParamFields - Required []string Collapsed bool Raw bool } @@ -76,9 +73,6 @@ type EntryConfig struct { Definitions []string ExpectedOutDir string TemplateFile string - InsertionPoint string - Collapsed bool - Raw bool } type ProcessOutput struct { @@ -115,9 +109,6 @@ func Process(ctx context.Context, p Entity) error { Definitions: defs, ExpectedOutDir: expectedOutDir, TemplateFile: expectedTemplateFile, - InsertionPoint: c.InsertionPoint, - Collapsed: c.Collapsed, - Raw: c.Raw, }) if err != nil { return err diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index f9048cd..b95cca4 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -40,7 +40,6 @@ type TerraformModule struct { } func NewTerraformModule(inputDir, outputDir, templatePath, insertionPoint string, collapsed, raw bool) Entity { - return &TerraformModule{ EntityConfig: EntityConfig{ InputDir: inputDir, @@ -63,8 +62,7 @@ func (t *TerraformModule) HandleEntries(ctx context.Context, c EntryConfig) (Pro Resources: make([]string, 0), } - for i := range c.Definitions { - path := c.Definitions[i] + for _, path := range c.Definitions { log.Printf("processing module at %s", path) mod, diag := tfconfig.LoadModule(path) if diag.HasErrors() { @@ -76,11 +74,18 @@ func (t *TerraformModule) HandleEntries(ctx context.Context, c EntryConfig) (Pro continue } - params, required := prepareParamsAndRequired(mod) + params := make(map[string]models.BackstageParamFields) + required := make([]string, 0) + for j := range mod.Variables { + params[j] = convertVariable(*mod.Variables[j]) + if mod.Variables[j].Required { + required = append(required, j) + } + } fileName := fmt.Sprintf("%s.yaml", filepath.Base(path)) outputFile := filepath.Join(c.ExpectedOutDir, fileName) - err := handleModuleOutput(ctx, outputFile, c.TemplateFile, c.InsertionPoint, params, required, c.Collapsed, c.Raw) + err := handleModuleOutput(ctx, outputFile, c.TemplateFile, t.InsertionPoint, params, required, t.Collapsed, t.Raw) if err != nil { return ProcessOutput{}, err } @@ -91,18 +96,6 @@ func (t *TerraformModule) HandleEntries(ctx context.Context, c EntryConfig) (Pro return out, nil } -func prepareParamsAndRequired(mod *tfconfig.Module) (map[string]models.BackstageParamFields, []string) { - params := make(map[string]models.BackstageParamFields) - required := make([]string, 0) - for j := range mod.Variables { - params[j] = convertVariable(*mod.Variables[j]) - if mod.Variables[j].Required { - required = append(required, j) - } - } - return params, required -} - func handleModuleOutput(ctx context.Context, outputFile, templateFile, insertionPoint string, properties map[string]models.BackstageParamFields, required []string, collapsed, raw bool) error { props := make(map[string]interface{}, len(properties)) for k := range properties { @@ -134,35 +127,6 @@ func handleModuleOutput(ctx context.Context, outputFile, templateFile, insertion return writeOutput(t, outputFile) } -func handleOutput(ctx context.Context, outputFile, templateFile, insertionPoint string, properties map[string]models.BackstageParamFields, required []string, collapsed, raw bool) error { - props := make(map[string]any, len(properties)) - for k := range properties { - props[k] = properties[k] - } - if !raw && !collapsed { - input := insertAtInput{ - templatePath: templateFile, - jqPathExpression: insertionPoint, - fields: map[string]any{ - "properties": props, - }, - } - if len(required) > 0 { - input.required = required - } - t, err := insertAt(ctx, input) - if err != nil { - return fmt.Errorf("failed to insert to given template: %s", err) - } - return writeOutput(t, outputFile) - } - t := map[string]any{ - "properties": props, - "required": required, - } - return writeOutput(t, outputFile) -} - func (t *TerraformModule) GetDefinitions(inputDir string, currentDepth uint32) ([]string, error) { if currentDepth > depth { return nil, nil @@ -239,27 +203,6 @@ func convertArray(tfVar tfconfig.Variable) models.BackstageParamFields { return out } -func convertObjectDefaults(tfVar tfconfig.Variable) map[string]*models.BackstageParamFields { - // build default values by taking default's key and type. Must be done for primitives only. - properties := make(map[string]*models.BackstageParamFields) - nestedType := getNestedType(cleanString(tfVar.Type)) - if tfVar.Default != nil { - d, ok := tfVar.Default.(map[string]any) - if !ok { - log.Fatalf("could not determine default type of %s\n", tfVar.Default) - } - for k := range d { - defaultProp := convertVariable(tfconfig.Variable{ - Name: k, - Type: nestedType, - Default: d[k], - }) - properties[k] = &defaultProp - } - } - return properties -} - func convertObject(tfVar tfconfig.Variable) models.BackstageParamFields { out := models.BackstageParamFields{ Title: tfVar.Name, @@ -299,14 +242,6 @@ func isPrimitive(s string) bool { return s == "string" || s == "number" || s == "bool" } -func isNestedPrimitive(s string) bool { - nested := strings.HasPrefix(s, "object(") || strings.HasPrefix(s, "map(") || strings.HasPrefix(s, "list(") - if nested { - return isPrimitive(getNestedType(s)) - } - return false -} - func getNestedType(s string) string { if strings.HasPrefix(s, "object(") { return strings.TrimSuffix(strings.SplitAfterN(s, "object(", 1)[1], ")") diff --git a/pkg/cmd/terraform_test.go b/pkg/cmd/terraform_test.go index ed66cef..6c08971 100644 --- a/pkg/cmd/terraform_test.go +++ b/pkg/cmd/terraform_test.go @@ -3,6 +3,7 @@ package cmd_test import ( "context" "fmt" + "log" "os" "path/filepath" @@ -10,12 +11,15 @@ import ( "github.com/cnoe-io/cnoe-cli/pkg/cmd" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" ) var _ = Describe("Terraform Template", func() { var ( tempDir string outputDir string + + stdout *gbytes.Buffer ) const ( @@ -37,6 +41,9 @@ var _ = Describe("Terraform Template", func() { outputDir = filepath.Join(tempDir, "output") err = os.Mkdir(outputDir, 0755) Expect(err).NotTo(HaveOccurred()) + + stdout = gbytes.NewBuffer() + log.SetOutput(stdout) }) AfterEach(func() { From e6651c841834b80527ccea33156d02d785956617 Mon Sep 17 00:00:00 2001 From: Nima Kaviani Date: Mon, 28 Aug 2023 22:14:05 -0700 Subject: [PATCH 14/15] further cleanup to the code --- pkg/cmd/crd.go | 49 +++++++++++++++++++++++--------------------- pkg/cmd/template.go | 2 +- pkg/cmd/terraform.go | 37 +++++++++++++++------------------ pkg/cmd/utils.go | 2 +- 4 files changed, 44 insertions(+), 46 deletions(-) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index ebaee20..7ceda4e 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -132,39 +132,42 @@ func (c *CRDModule) HandleEntries(ctx context.Context, cc EntryConfig) (ProcessO return ProcessOutput{}, err } - filename := filepath.Join(cc.ExpectedOutDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) - - if !c.Collapsed && !c.Raw { - input := insertAtInput{ - templatePath: cc.TemplateFile, - jqPathExpression: c.InsertionPoint, - } - props := converted.(map[string]any) - if v, reqOk := props["required"]; reqOk { - if reqs, ok := v.([]string); ok { - input.required = reqs - } - } - input.fields = props - t, err := insertAt(ctx, input) - if err != nil { - return ProcessOutput{}, err - } - converted = t - } - - err = writeOutput(converted, filename) + fileName := filepath.Join(cc.ExpectedOutDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) + c.handleModuleOuptut(ctx, converted, fileName, cc.TemplateFile) if err != nil { log.Printf("failed to write %s: %s \n", def, err.Error()) return ProcessOutput{}, err } - out.Templates = append(out.Templates, filename) + out.Templates = append(out.Templates, fileName) out.Resources = append(out.Resources, resourceName) } return out, nil } +func (c *CRDModule) handleModuleOuptut(ctx context.Context, converted any, outputFile, templateFile string) error { + if !c.Collapsed && !c.Raw { + input := insertAtInput{ + templatePath: templateFile, + jqPathExpression: c.InsertionPoint, + } + props := converted.(map[string]any) + if v, reqOk := props["required"]; reqOk { + if reqs, ok := v.([]string); ok { + input.required = reqs + } + } + input.fields = props + converted, err := insertAt(ctx, input) + if err != nil { + return err + } + return writeOutput(converted, outputFile) + } + + return writeOutput(converted, outputFile) +} + func convert(def string) (any, string, error) { data, err := os.ReadFile(def) if err != nil { diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index f769314..554237a 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -120,7 +120,7 @@ func Process(ctx context.Context, p Entity) error { templatePath: expectedTemplateFile, jqPathExpression: c.InsertionPoint, } - return writeOneOf(ctx, input, generatedTemplateFile, output.Templates) + return writeCollapsedTemplate(ctx, input, generatedTemplateFile, output.Templates) } return nil diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index b95cca4..3c62530 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -62,15 +62,15 @@ func (t *TerraformModule) HandleEntries(ctx context.Context, c EntryConfig) (Pro Resources: make([]string, 0), } - for _, path := range c.Definitions { - log.Printf("processing module at %s", path) - mod, diag := tfconfig.LoadModule(path) + for _, def := range c.Definitions { + log.Printf("processing module at %s", def) + mod, diag := tfconfig.LoadModule(def) if diag.HasErrors() { return out, diag.Err() } if len(mod.Variables) == 0 { - log.Printf("module %s does not have variables", path) + log.Printf("module %s does not have variables", def) continue } @@ -83,10 +83,10 @@ func (t *TerraformModule) HandleEntries(ctx context.Context, c EntryConfig) (Pro } } - fileName := fmt.Sprintf("%s.yaml", filepath.Base(path)) - outputFile := filepath.Join(c.ExpectedOutDir, fileName) - err := handleModuleOutput(ctx, outputFile, c.TemplateFile, t.InsertionPoint, params, required, t.Collapsed, t.Raw) + fileName := filepath.Join(c.ExpectedOutDir, fmt.Sprintf("%s.yaml", filepath.Base(def))) + err := t.handleModuleOutput(ctx, fileName, c.TemplateFile, params, required) if err != nil { + log.Printf("failed to write %s: %s \n", def, err.Error()) return ProcessOutput{}, err } @@ -96,35 +96,30 @@ func (t *TerraformModule) HandleEntries(ctx context.Context, c EntryConfig) (Pro return out, nil } -func handleModuleOutput(ctx context.Context, outputFile, templateFile, insertionPoint string, properties map[string]models.BackstageParamFields, required []string, collapsed, raw bool) error { - props := make(map[string]interface{}, len(properties)) - for k := range properties { - props[k] = properties[k] - } - - if !raw && !collapsed { +func (t *TerraformModule) handleModuleOutput(ctx context.Context, outputFile, templateFile string, properties map[string]models.BackstageParamFields, required []string) error { + if !t.Raw && !t.Collapsed { input := insertAtInput{ - templatePath: templateFile, + templatePath: t.TemplateFile, jqPathExpression: insertionPoint, fields: map[string]interface{}{ - "properties": props, + "properties": properties, }, } if len(required) > 0 { input.required = required } - t, err := insertAt(ctx, input) + content, err := insertAt(ctx, input) if err != nil { return fmt.Errorf("failed to insert to given template: %s", err) } - return writeOutput(t, outputFile) + return writeOutput(content, outputFile) } - t := map[string]interface{}{ - "properties": props, + content := map[string]interface{}{ + "properties": properties, "required": required, } - return writeOutput(t, outputFile) + return writeOutput(content, outputFile) } func (t *TerraformModule) GetDefinitions(inputDir string, currentDepth uint32) ([]string, error) { diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go index e7cda67..307792d 100644 --- a/pkg/cmd/utils.go +++ b/pkg/cmd/utils.go @@ -81,7 +81,7 @@ func prepDirectories(inputDir, outputDir, templateFile string, oneOf bool) (stri // Use the given template file, add dependencies and enum fields at the object specified by insertionPoint. // Write the result to a file specified by outputFile. -func writeOneOf(ctx context.Context, input insertAtInput, outputFile string, resourceFiles []string) error { +func writeCollapsedTemplate(ctx context.Context, input insertAtInput, outputFile string, resourceFiles []string) error { t, err := oneOf(ctx, resourceFiles, input) if err != nil { From c6dabc573339e591b185febf371a589a46236d88 Mon Sep 17 00:00:00 2001 From: Nima Kaviani Date: Tue, 29 Aug 2023 00:25:00 -0700 Subject: [PATCH 15/15] combine more of similar code into one place --- pkg/cmd/crd.go | 50 +++++++++++++++-------------------- pkg/cmd/crd_test.go | 7 +++++ pkg/cmd/template.go | 49 ++++++++++++++++------------------ pkg/cmd/terraform.go | 63 +++++++++++++++++++------------------------- pkg/cmd/utils.go | 8 ++++++ 5 files changed, 86 insertions(+), 91 deletions(-) diff --git a/pkg/cmd/crd.go b/pkg/cmd/crd.go index 7ceda4e..35c8a09 100644 --- a/pkg/cmd/crd.go +++ b/pkg/cmd/crd.go @@ -116,37 +116,29 @@ func (c *CRDModule) findDefs(file os.DirEntry, currentDepth uint32, base string) return []string{f}, nil } -func (c *CRDModule) HandleEntries(ctx context.Context, cc EntryConfig) (ProcessOutput, error) { - out := ProcessOutput{ - Templates: make([]string, 0), - Resources: make([]string, 0), - } - - for _, def := range cc.Definitions { - converted, resourceName, err := convert(def) - if err != nil { - var e NotSupported - if errors.As(err, &e) { - continue - } - return ProcessOutput{}, err +func (c *CRDModule) HandleEntry(ctx context.Context, def, expectedOutDir, templateFile string) (any, string, error) { + log.Printf("processing resource at %s", def) + converted, resourceName, err := convert(def) + if err != nil { + var e NotSupported + if errors.As(err, &e) { + return nil, "", nil } + return nil, "", err + } - fileName := filepath.Join(cc.ExpectedOutDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) - c.handleModuleOuptut(ctx, converted, fileName, cc.TemplateFile) - if err != nil { - log.Printf("failed to write %s: %s \n", def, err.Error()) - return ProcessOutput{}, err - } - out.Templates = append(out.Templates, fileName) - out.Resources = append(out.Resources, resourceName) + fileName := filepath.Join(expectedOutDir, fmt.Sprintf("%s.yaml", strings.ToLower(resourceName))) + content, err := c.createContent(ctx, converted, templateFile) + if err != nil { + log.Printf("failed to write %s: %s \n", def, err.Error()) + return nil, "", err } - return out, nil + return content, fileName, nil } -func (c *CRDModule) handleModuleOuptut(ctx context.Context, converted any, outputFile, templateFile string) error { - if !c.Collapsed && !c.Raw { +func (c *CRDModule) createContent(ctx context.Context, converted any, templateFile string) (any, error) { + if shouldCreateNonCollapsedTemplate(c) { input := insertAtInput{ templatePath: templateFile, jqPathExpression: c.InsertionPoint, @@ -160,12 +152,12 @@ func (c *CRDModule) handleModuleOuptut(ctx context.Context, converted any, outpu input.fields = props converted, err := insertAt(ctx, input) if err != nil { - return err + return nil, err } - return writeOutput(converted, outputFile) + return converted, nil } - return writeOutput(converted, outputFile) + return converted, nil } func convert(def string) (any, string, error) { @@ -203,7 +195,7 @@ func convert(def string) (any, string, error) { } else { value, err = ConvertMap(v) if err != nil { - return ProcessOutput{}, "", err + return nil, "", err } } diff --git a/pkg/cmd/crd_test.go b/pkg/cmd/crd_test.go index d94609d..8990ea5 100644 --- a/pkg/cmd/crd_test.go +++ b/pkg/cmd/crd_test.go @@ -3,18 +3,22 @@ package cmd_test import ( "context" "fmt" + "log" "os" "path/filepath" "github.com/cnoe-io/cnoe-cli/pkg/cmd" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" ) var _ = Describe("Template CRDs", func() { var ( tempDir string outputDir string + + stdout *gbytes.Buffer ) const ( @@ -37,6 +41,9 @@ var _ = Describe("Template CRDs", func() { outputDir = filepath.Join(tempDir, "output") err = os.Mkdir(outputDir, 0755) Expect(err).NotTo(HaveOccurred()) + + stdout = gbytes.NewBuffer() + log.SetOutput(stdout) }) AfterEach(func() { diff --git a/pkg/cmd/template.go b/pkg/cmd/template.go index 554237a..ee6e763 100644 --- a/pkg/cmd/template.go +++ b/pkg/cmd/template.go @@ -48,11 +48,11 @@ func templatePreRunE(cmd *cobra.Command, args []string) error { } if collapsed && templatePath == "" && !raw { - return errors.New("templatePath flag must be specified when using the `collapse` flag (optionally you can use `insertAt` as well.)") + return errors.New("templatePath flag must be specified when using the `collapse` flag (optionally you can use `insertAt` as well)") } if templatePath == "" && !raw { - return errors.New("you either need to use the `raw` flag to generate raw OpenAPI files or define a `templatePath` for the tool to populate.") + return errors.New("you either need to use the `raw` flag to generate raw OpenAPI files or define a `templatePath` for the tool to populate") } return nil @@ -69,20 +69,9 @@ type EntityConfig struct { Raw bool } -type EntryConfig struct { - Definitions []string - ExpectedOutDir string - TemplateFile string -} - -type ProcessOutput struct { - Templates []string - Resources []string -} - type Entity interface { GetDefinitions(string, uint32) ([]string, error) - HandleEntries(context.Context, EntryConfig) (ProcessOutput, error) + HandleEntry(context.Context, string, string, string) (any, string, error) Config() EntityConfig } @@ -93,34 +82,42 @@ func Process(ctx context.Context, p Entity) error { c.InputDir, c.OutputDir, c.TemplateFile, - c.Collapsed && !c.Raw, /* only generate nesting if needs to be collapsed and not be raw*/ + c.Collapsed && !c.Raw, /* only generate nesting if templates need to collapse and not printed as raw*/ ) if err != nil { return err } - defs, err := p.GetDefinitions(expectedInDir, 0) + definitions, err := p.GetDefinitions(expectedInDir, 0) if err != nil { return err } - log.Printf("processing %d definitions", len(defs)) + log.Printf("processing %d definitions", len(definitions)) - output, err := p.HandleEntries(ctx, EntryConfig{ - Definitions: defs, - ExpectedOutDir: expectedOutDir, - TemplateFile: expectedTemplateFile, - }) - if err != nil { - return err + templateOutputFiles := make([]string, 0) + + for _, def := range definitions { + content, contentFileName, err := p.HandleEntry(ctx, def, expectedOutDir, expectedTemplateFile) + if err != nil { + return err + } + if content != nil { // write the content and record the file name + err = writeOutput(content, contentFileName) + if err != nil { + log.Printf("wrinting content failed for %s", contentFileName) + continue + } + templateOutputFiles = append(templateOutputFiles, contentFileName) + } } - if c.Collapsed && !c.Raw { + if shouldCreateCollapsedTemplate(p) && len(templateOutputFiles) > 0 { generatedTemplateFile := filepath.Join(expectedOutDir, "../template.yaml") input := insertAtInput{ templatePath: expectedTemplateFile, jqPathExpression: c.InsertionPoint, } - return writeCollapsedTemplate(ctx, input, generatedTemplateFile, output.Templates) + return writeCollapsedTemplate(ctx, input, generatedTemplateFile, templateOutputFiles) } return nil diff --git a/pkg/cmd/terraform.go b/pkg/cmd/terraform.go index 3c62530..24717e3 100644 --- a/pkg/cmd/terraform.go +++ b/pkg/cmd/terraform.go @@ -56,48 +56,39 @@ func (t *TerraformModule) Config() EntityConfig { return t.EntityConfig } -func (t *TerraformModule) HandleEntries(ctx context.Context, c EntryConfig) (ProcessOutput, error) { - out := ProcessOutput{ - Templates: make([]string, 0), - Resources: make([]string, 0), +func (t *TerraformModule) HandleEntry(ctx context.Context, def, expectedOutDir, templateFile string) (any, string, error) { + log.Printf("processing module at %s", def) + mod, diag := tfconfig.LoadModule(def) + if diag.HasErrors() { + return nil, "", diag.Err() } - for _, def := range c.Definitions { - log.Printf("processing module at %s", def) - mod, diag := tfconfig.LoadModule(def) - if diag.HasErrors() { - return out, diag.Err() - } - - if len(mod.Variables) == 0 { - log.Printf("module %s does not have variables", def) - continue - } - - params := make(map[string]models.BackstageParamFields) - required := make([]string, 0) - for j := range mod.Variables { - params[j] = convertVariable(*mod.Variables[j]) - if mod.Variables[j].Required { - required = append(required, j) - } - } + if len(mod.Variables) == 0 { + log.Printf("module %s does not have variables", def) + return nil, "", nil + } - fileName := filepath.Join(c.ExpectedOutDir, fmt.Sprintf("%s.yaml", filepath.Base(def))) - err := t.handleModuleOutput(ctx, fileName, c.TemplateFile, params, required) - if err != nil { - log.Printf("failed to write %s: %s \n", def, err.Error()) - return ProcessOutput{}, err + params := make(map[string]models.BackstageParamFields) + required := make([]string, 0) + for j := range mod.Variables { + params[j] = convertVariable(*mod.Variables[j]) + if mod.Variables[j].Required { + required = append(required, j) } + } - out.Templates = append(out.Templates, fileName) + fileName := filepath.Join(expectedOutDir, fmt.Sprintf("%s.yaml", filepath.Base(def))) + content, err := t.createContent(ctx, templateFile, params, required) + if err != nil { + log.Printf("failed to write %s: %s \n", def, err.Error()) + return nil, "", err } - return out, nil + return content, fileName, nil } -func (t *TerraformModule) handleModuleOutput(ctx context.Context, outputFile, templateFile string, properties map[string]models.BackstageParamFields, required []string) error { - if !t.Raw && !t.Collapsed { +func (t *TerraformModule) createContent(ctx context.Context, templateFile string, properties map[string]models.BackstageParamFields, required []string) (any, error) { + if shouldCreateNonCollapsedTemplate(t) { input := insertAtInput{ templatePath: t.TemplateFile, jqPathExpression: insertionPoint, @@ -110,16 +101,16 @@ func (t *TerraformModule) handleModuleOutput(ctx context.Context, outputFile, te } content, err := insertAt(ctx, input) if err != nil { - return fmt.Errorf("failed to insert to given template: %s", err) + return nil, fmt.Errorf("failed to insert to given template: %s", err) } - return writeOutput(content, outputFile) + return content, nil } content := map[string]interface{}{ "properties": properties, "required": required, } - return writeOutput(content, outputFile) + return content, nil } func (t *TerraformModule) GetDefinitions(inputDir string, currentDepth uint32) ([]string, error) { diff --git a/pkg/cmd/utils.go b/pkg/cmd/utils.go index 307792d..be495bf 100644 --- a/pkg/cmd/utils.go +++ b/pkg/cmd/utils.go @@ -199,3 +199,11 @@ func getRelevantFiles(inputDir string, currentDepth uint32, f finder) ([]string, } return out, nil } + +func shouldCreateCollapsedTemplate(p Entity) bool { + return p.Config().Collapsed && !p.Config().Raw +} + +func shouldCreateNonCollapsedTemplate(p Entity) bool { + return !p.Config().Collapsed && !p.Config().Raw +}