diff --git a/.gitignore b/.gitignore index 8c750ea..a48f74d 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,5 @@ Icon Network Trash Folder Temporary Items .apdisk +hcl +tf diff --git a/cli/args.go b/cli/args.go index 62aab1f..5c7db8f 100644 --- a/cli/args.go +++ b/cli/args.go @@ -1,23 +1,36 @@ package cli import ( + "errors" "flag" - "log" + "fmt" "os" + + "github.com/env0/terratag/internal/common" ) type Args struct { Tags string Dir string Filter string + Type string IsSkipTerratagFiles bool Verbose bool Rename bool } -func InitArgs() (Args, bool) { +func validate(args Args) error { + if args.Tags == "" { + return errors.New("missing tags") + } + if args.Type != string(common.Terraform) && args.Type != string(common.Terragrunt) { + return fmt.Errorf("invalid type %s, must be either 'terratag' or 'terragrunt'", args.Type) + } + return nil +} + +func InitArgs() (Args, error) { args := Args{} - isMissingArg := false programName := os.Args[0] programArgs := os.Args[1:] @@ -29,13 +42,15 @@ func InitArgs() (Args, bool) { fs.StringVar(&args.Filter, "filter", ".*", "Only apply tags to the selected resource types (regex)") fs.BoolVar(&args.Verbose, "verbose", false, "Enable verbose logging") fs.BoolVar(&args.Rename, "rename", true, "Keep the original filename or replace it with .terratag.tf") + fs.StringVar(&args.Type, "type", string(common.Terraform), "The IAC type. Valid values: terraform or terragrunt") - err := fs.Parse(programArgs) + if err := fs.Parse(programArgs); err != nil { + return args, err + } - if err != nil || args.Tags == "" { - log.Println("Usage: terratag -tags='{ \"some_tag\": \"value\" }' [-dir=\".\"]") - isMissingArg = true + if err := validate(args); err != nil { + return args, err } - return args, isMissingArg + return args, nil } diff --git a/cmd/terratag/main.go b/cmd/terratag/main.go index deef53e..7c9b46a 100644 --- a/cmd/terratag/main.go +++ b/cmd/terratag/main.go @@ -11,13 +11,17 @@ import ( ) func main() { - args, isMissingArg := cli.InitArgs() - if isMissingArg { + args, err := cli.InitArgs() + if err != nil { + log.Println(err) + log.Println("Usage: terratag -tags='{ \"some_tag\": \"value\" }' [-dir=\".\"]") return } initLogFiltering(args.Verbose) - terratag.Terratag(args) + if err := terratag.Terratag(args); err != nil { + log.Printf("[ERROR] execution failed due to an error\n%v", err) + } } func initLogFiltering(verbose bool) { diff --git a/internal/common/common.go b/internal/common/common.go new file mode 100644 index 0000000..b222679 --- /dev/null +++ b/internal/common/common.go @@ -0,0 +1,31 @@ +package common + +import "github.com/hashicorp/hcl/v2/hclwrite" + +type IACType string + +const ( + Terraform IACType = "terraform" + Terragrunt IACType = "terragrunt" +) + +type Version struct { + Major int + Minor int +} + +type TaggingArgs struct { + Filter string + Dir string + Tags string + Matches []string + IsSkipTerratagFiles bool + Rename bool + IACType IACType + TFVersion Version +} + +type TerratagLocal struct { + Found map[string]hclwrite.Tokens + Added string +} diff --git a/internal/convert/convert.go b/internal/convert/convert.go index 6fc707a..9c72279 100644 --- a/internal/convert/convert.go +++ b/internal/convert/convert.go @@ -6,6 +6,7 @@ import ( "log" "strings" + "github.com/env0/terratag/internal/common" "github.com/env0/terratag/internal/tag_keys" "github.com/env0/terratag/internal/utils" "github.com/hashicorp/hcl/v2/hclsyntax" @@ -14,7 +15,7 @@ import ( "github.com/zclconf/go-cty/cty" ) -func GetExistingTagsExpression(tokens hclwrite.Tokens, tfVersion Version) string { +func GetExistingTagsExpression(tokens hclwrite.Tokens, tfVersion common.Version) string { // NOTE: consider removing buildMapExpression in case tf v0.11.0 support is removed. if isHclMap(tokens) && tfVersion.Major == 0 && tfVersion.Minor <= 11 { return buildMapExpression(tokens, tfVersion) @@ -51,7 +52,7 @@ func trimTokens(tokens hclwrite.Tokens) hclwrite.Tokens { return tokens[startIndex : len(tokens)-endIndex] } -func buildMapExpression(tokens hclwrite.Tokens, tfVersion Version) string { +func buildMapExpression(tokens hclwrite.Tokens, tfVersion common.Version) string { if tfVersion.Major == 0 && tfVersion.Minor >= 15 || tfVersion.Major == 1 { mapContent := strings.TrimSpace(string(tokens.Bytes())) return "tomap(" + mapContent + ")" @@ -115,7 +116,7 @@ func stringifyExpression(tokens hclwrite.Tokens) string { return expression } -func AppendLocalsBlock(file *hclwrite.File, filename string, terratag TerratagLocal) { +func AppendLocalsBlock(file *hclwrite.File, filename string, terratag common.TerratagLocal) { key := tag_keys.GetTerratagAddedKey(filename) // If there's an existings terratag locals replace it with the merged locals. @@ -176,7 +177,7 @@ func UnquoteTagsAttribute(swappedTagsStrings []string, text string) string { return text } -func MoveExistingTags(filename string, terratag TerratagLocal, block *hclwrite.Block, tagId string) (bool, error) { +func MoveExistingTags(filename string, terratag common.TerratagLocal, block *hclwrite.Block, tagId string) (bool, error) { var existingTags hclwrite.Tokens // First we try to find tags as attribute @@ -256,12 +257,3 @@ func quoteAttributeKeys(tagsAttribute *hclwrite.Attribute) hclwrite.Tokens { return newTags } - -type TerratagLocal struct { - Found map[string]hclwrite.Tokens - Added string -} -type Version struct { - Major int - Minor int -} diff --git a/internal/tagging/tagging.go b/internal/tagging/tagging.go index 31e093f..d681e78 100644 --- a/internal/tagging/tagging.go +++ b/internal/tagging/tagging.go @@ -3,6 +3,7 @@ package tagging import ( "log" + "github.com/env0/terratag/internal/common" "github.com/env0/terratag/internal/convert" "github.com/env0/terratag/internal/tag_keys" "github.com/env0/terratag/internal/terraform" @@ -80,9 +81,9 @@ type TagBlockArgs struct { Filename string Block *hclwrite.Block Tags string - Terratag convert.TerratagLocal + Terratag common.TerratagLocal TagId string - TfVersion convert.Version + TfVersion common.Version } type TagResourceFn func(args TagBlockArgs) (*Result, error) diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index edefbdb..d57f2e9 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "io/ioutil" "log" "os" @@ -14,12 +15,12 @@ import ( "strings" "github.com/bmatcuk/doublestar" - "github.com/env0/terratag/internal/convert" + "github.com/env0/terratag/internal/common" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/thoas/go-funk" ) -func GetTerraformVersion() (*convert.Version, error) { +func GetTerraformVersion() (*common.Version, error) { output, err := exec.Command("terraform", "version").Output() if err != nil { return nil, err @@ -46,7 +47,7 @@ func GetTerraformVersion() (*convert.Version, error) { return nil, fmt.Errorf("terratag only supports Terraform from version 0.11.x and up to 1.2.x - your version says %s", outputAsString) } - return &convert.Version{Major: majorVersion, Minor: minorVersion}, nil + return &common.Version{Major: majorVersion, Minor: minorVersion}, nil } type VersionPart int @@ -73,21 +74,59 @@ func GetResourceType(resource hclwrite.Block) string { return resource.Labels()[0] } -func ValidateTerraformInitRun(dir string) error { - _, err := os.Stat(dir + "/.terraform") +func getRootDir(iacType string) string { + if iacType == string(common.Terragrunt) { + return "/.terragrunt-cache" + } else { + return "/.terraform" + } +} - if err != nil { +func ValidateInitRun(dir string, iacType string) error { + path := dir + getRootDir(iacType) + + if _, err := os.Stat(path); err != nil { if os.IsNotExist(err) { - return errors.New("terraform init must run before running terratag") + return fmt.Errorf("%s init must run before running terratag", iacType) } - return fmt.Errorf("couldn't determine if terraform init has run: %v", err) + return fmt.Errorf("couldn't determine if %s init has run: %v", iacType, err) } return nil } -func GetTerraformFilePaths(rootDir string) ([]string, error) { +func GetFilePaths(dir string, iacType string) ([]string, error) { + if iacType == string(common.Terragrunt) { + return getTerragruntFilePath(dir) + } else { + return getTerraformFilePaths(dir) + } +} + +func getTerragruntFilePath(rootDir string) ([]string, error) { + rootDir += getRootDir(string(common.Terragrunt)) + + var tfFiles []string + if err := filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + log.Printf("[WARN] skipping %s due to an error: %v", path, err) + return filepath.SkipDir + } + + if strings.HasSuffix(path, ".tf") { + tfFiles = append(tfFiles, path) + } + + return nil + }); err != nil { + return nil, err + } + + return tfFiles, nil +} + +func getTerraformFilePaths(rootDir string) ([]string, error) { const tfFileMatcher = "/*.tf" tfFiles, err := doublestar.Glob(rootDir + tfFileMatcher) diff --git a/internal/tfschema/tfschema.go b/internal/tfschema/tfschema.go index 9f3eb9a..0278786 100644 --- a/internal/tfschema/tfschema.go +++ b/internal/tfschema/tfschema.go @@ -2,10 +2,13 @@ package tfschema import ( "fmt" + "io/fs" "log" + "path/filepath" "strings" "sync" + "github.com/env0/terratag/internal/common" "github.com/env0/terratag/internal/providers" "github.com/env0/terratag/internal/tagging" "github.com/env0/terratag/internal/terraform" @@ -21,13 +24,13 @@ var providerToClientMapLock sync.Mutex var customSupportedProviderNames = [...]string{"google-beta"} -func IsTaggable(dir string, resource hclwrite.Block) (bool, error) { +func IsTaggable(dir string, iacType common.IACType, resource hclwrite.Block) (bool, error) { var isTaggable bool resourceType := terraform.GetResourceType(resource) if providers.IsSupportedResource(resourceType) { providerName, _ := detectProviderName(resource) - client, err := getClient(providerName, dir) + client, err := getClient(providerName, dir, iacType) if err != nil { return false, err } @@ -84,7 +87,33 @@ func detectProviderName(resource hclwrite.Block) (string, error) { return extractProviderNameFromResourceType(terraform.GetResourceType(resource)) } -func getClient(providerName string, dir string) (tfschema.Client, error) { +func getTerragruntPluginPath(dir string) string { + dir += "/.terragrunt-cache" + ret := dir + found := false + + filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if found || err != nil { + return filepath.SkipDir + } + + // E.g. ./.terragrunt-cache/yHtqnMrVQOISIYxobafVvZbAAyU/ThyYwttwki6d6AS3aD5OwoyqIWA/.terraform + if strings.HasSuffix(path, "/.terraform") { + ret = strings.TrimSuffix(path, "/.terraform") + found = true + } + + return nil + }) + + return ret +} + +func getClient(providerName string, dir string, iacType common.IACType) (tfschema.Client, error) { + if iacType == common.Terragrunt { + dir = getTerragruntPluginPath(dir) + } + providerToClientMapLock.Lock() defer providerToClientMapLock.Unlock() diff --git a/terratag.go b/terratag.go index 8d40b12..17cd69e 100644 --- a/terratag.go +++ b/terratag.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "github.com/env0/terratag/cli" + "github.com/env0/terratag/internal/common" "github.com/env0/terratag/internal/convert" "github.com/env0/terratag/internal/file" "github.com/env0/terratag/internal/providers" @@ -45,16 +46,27 @@ func Terratag(args cli.Args) error { return err } - if err := terraform.ValidateTerraformInitRun(args.Dir); err != nil { + if err := terraform.ValidateInitRun(args.Dir, args.Type); err != nil { return err } - matches, err := terraform.GetTerraformFilePaths(args.Dir) + matches, err := terraform.GetFilePaths(args.Dir, args.Type) if err != nil { return err } - counters := tagDirectoryResources(args.Dir, args.Filter, matches, args.Tags, args.IsSkipTerratagFiles, *tfVersion, args.Rename) + taggingArgs := &common.TaggingArgs{ + Filter: args.Filter, + Dir: args.Dir, + Tags: args.Tags, + Matches: matches, + IsSkipTerratagFiles: args.IsSkipTerratagFiles, + Rename: args.Rename, + IACType: common.IACType(args.Type), + TFVersion: *tfVersion, + } + + counters := tagDirectoryResources(taggingArgs) log.Print("[INFO] Summary:") log.Print("[INFO] Tagged ", counters.taggedResources, " resource/s (out of ", counters.totalResources, " resource/s processed)") log.Print("[INFO] In ", counters.taggedFiles, " file/s (out of ", counters.totalFiles, " file/s processed)") @@ -62,10 +74,11 @@ func Terratag(args cli.Args) error { return nil } -func tagDirectoryResources(dir string, filter string, matches []string, tags string, isSkipTerratagFiles bool, tfVersion convert.Version, rename bool) counters { +// dir string, filter string, matches []string, tags string, isSkipTerratagFiles bool, tfVersion convert.Version, rename bool, iacType string +func tagDirectoryResources(args *common.TaggingArgs) counters { var total counters - for _, path := range matches { - if isSkipTerratagFiles && strings.HasSuffix(path, "terratag.tf") { + for _, path := range args.Matches { + if args.IsSkipTerratagFiles && strings.HasSuffix(path, "terratag.tf") { log.Print("[INFO] Skipping file ", path, " as it's already tagged") } else { matchWaitGroup.Add(1) @@ -83,7 +96,7 @@ func tagDirectoryResources(dir string, filter string, matches []string, tags str } }() - perFile, err := tagFileResources(path, dir, filter, tags, tfVersion, rename) + perFile, err := tagFileResources(path, args) if err != nil { log.Printf("[ERROR] failed to process %s due to an error\n%v", path, err) return @@ -99,7 +112,7 @@ func tagDirectoryResources(dir string, filter string, matches []string, tags str return total } -func tagFileResources(path string, dir string, filter string, tags string, tfVersion convert.Version, rename bool) (*counters, error) { +func tagFileResources(path string, args *common.TaggingArgs) (*counters, error) { perFileCounters := counters{} log.Print("[INFO] Processing file ", path) var swappedTagsStrings []string @@ -111,12 +124,12 @@ func tagFileResources(path string, dir string, filter string, tags string, tfVer filename := file.GetFilename(path) - hclMap, err := toHclMap(tags) + hclMap, err := toHclMap(args.Tags) if err != nil { return nil, err } - terratag := convert.TerratagLocal{ + terratag := common.TerratagLocal{ Found: map[string]hclwrite.Tokens{}, Added: hclMap, } @@ -127,7 +140,7 @@ func tagFileResources(path string, dir string, filter string, tags string, tfVer log.Print("[INFO] Processing resource ", resource.Labels()) perFileCounters.totalResources += 1 - matched, err := regexp.MatchString(filter, resource.Labels()[0]) + matched, err := regexp.MatchString(args.Filter, resource.Labels()[0]) if err != nil { return nil, err } @@ -136,7 +149,7 @@ func tagFileResources(path string, dir string, filter string, tags string, tfVer continue } - isTaggable, err := tfschema.IsTaggable(dir, *resource) + isTaggable, err := tfschema.IsTaggable(args.Dir, args.IACType, *resource) if err != nil { return nil, err } @@ -147,10 +160,10 @@ func tagFileResources(path string, dir string, filter string, tags string, tfVer result, err := tagging.TagResource(tagging.TagBlockArgs{ Filename: filename, Block: resource, - Tags: tags, + Tags: args.Tags, Terratag: terratag, TagId: providers.GetTagIdByResource(terraform.GetResourceType(*resource)), - TfVersion: tfVersion, + TfVersion: args.TFVersion, }) if err != nil { return nil, err @@ -189,7 +202,7 @@ func tagFileResources(path string, dir string, filter string, tags string, tfVer swappedTagsStrings = append(swappedTagsStrings, terratag.Added) text = convert.UnquoteTagsAttribute(swappedTagsStrings, text) - if err := file.ReplaceWithTerratagFile(path, text, rename); err != nil { + if err := file.ReplaceWithTerratagFile(path, text, args.Rename); err != nil { return nil, err } perFileCounters.taggedFiles = 1 diff --git a/terratag_test.go b/terratag_test.go index ef5de2d..db30117 100644 --- a/terratag_test.go +++ b/terratag_test.go @@ -2,7 +2,6 @@ package terratag import ( "bytes" - "errors" "fmt" "io/ioutil" "log" @@ -241,11 +240,11 @@ func run_terratag(entryDir string, filter string) (err interface{}) { } else { os.Args = append(args, "-dir="+entryDir, "-filter="+filter) } - args, isMissingArg := cli.InitArgs() + args, err := cli.InitArgs() os.Args = cleanArgs osArgsLock.Unlock() - if isMissingArg { - return errors.New("Missing arg") + if err != nil { + return err } Terratag(args)