diff --git a/internal/tfimport/tfimport.go b/internal/tfimport/tfimport.go index 345ddcb13..7992245eb 100644 --- a/internal/tfimport/tfimport.go +++ b/internal/tfimport/tfimport.go @@ -28,6 +28,7 @@ import ( "github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/fileutil" "github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/runner" + "github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/template" "github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/terraform" "github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/tfimport/importer" ) @@ -517,6 +518,40 @@ func DoesNotExist(output string) bool { return reDoesNotExist.FindStringIndex(output) != nil } +// importError is an error indicating a resource failed to be imported. +// This doesn't indicate an error in running the steps around trying an import, +// it's specifically for terraform failing to import a resource for a reason +// that can't be handled (i.e. does not exist or not importable at all). +type importError struct { + // errMgs contains the error messages for all failed imports. + errMsgs []string +} + +func (e *importError) Error() string { + return fmt.Sprintf("failed to find or import %v resources. Use --verbose flag to see detailed error messages.", len(e.errMsgs)) +} + +var summaryTmpl string = ` +Summary: + +Found {{.total}} importable resources + +Successfully imported {{len .successes}}{{if gt (len .successes) 0}}:{{end}} +{{- range $resource := .successes}} +- {{$resource}} +{{- end}} + +Skipped {{len .skipped}}{{if gt (len .skipped) 0}}:{{end}} +{{- range $resource := .skipped}} +- {{$resource}} +{{- end}} + +Failed to import {{len .failures}}{{if gt (len .failures) 0}}:{{end}} +{{- range $resource := .failures}} +- {{$resource}} +{{- end}} +` + // RunArgs are the supported tfimport run arguments. type RunArgs struct { InputDir string @@ -538,21 +573,53 @@ func Run(rn runner.Runner, importRn runner.Runner, runArgs *RunArgs) error { return fmt.Errorf("expand path %q: %v", inputDir, err) } + var successesTotal, failuresTotal []string skipped := make(map[string]bool) + var ie *importError for { - retry, err := planAndImport(rn, importRn, runArgs, skipped) - if err != nil { + failuresTotal = nil + successes, err := planAndImport(rn, importRn, runArgs, skipped) + successesTotal = append(successesTotal, successes...) + + if err == nil { + // Either no successes, or no errors. + // No need to retry. + break + } + + if !errors.As(err, &ie) { + // It's some other kind of error and we should return it. return err } - // If all imports succeeded, or none did, stop trying to import things, since we aren't making progress anymore. - if !retry { + // It's an importError. + failuresTotal = ie.errMsgs + + // If no successes, break out of the loop. + if len(successes) <= 0 { break } + // Some resources imported successfully, but others didn't. + // Time to retry. log.Println("Some imports succeeded but others did not. Retrying the import, in case dependent values have now been populated.") } + buf, err := template.WriteBuffer(summaryTmpl, map[string]interface{}{ + "total": len(successesTotal) + len(skipped) + len(failuresTotal), + "successes": successesTotal, + "skipped": skipped, + "failures": failuresTotal, + }) + if err != nil { + return fmt.Errorf("building summary template: %v", err) + } + log.Printf("%s\n", buf) + + if ie != nil { + return ie + } + return nil } @@ -560,7 +627,7 @@ func Run(rn runner.Runner, importRn runner.Runner, runArgs *RunArgs) error { // If it imported some resources but failed to import others, it will return true for retry. This is a simple way to solve dependencies without having to figure out the graph. // A specific case: GKE node pool name depends on random_id; import the random_id first, then do the cycle again and import the node pool. // skipped is a map to be filled with skipped resources so they are skipped on subsequent runs too. -func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[string]bool) (retry bool, err error) { +func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[string]bool) (successes []string, err error) { // Create Terraform command runners. tfCmdOutput := func(args ...string) ([]byte, error) { cmd := exec.Command(runArgs.TerraformPath, args...) @@ -570,36 +637,35 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str // Init is safe to run on an already-initialized config dir. if out, err := tfCmdOutput("init"); err != nil { - return false, fmt.Errorf("init: %v\v%v", err, string(out)) + return nil, fmt.Errorf("init: %v\v%v", err, string(out)) } // Generate and load the plan using a temp file. // Use the .tf files input dir in case the system can't write to /tmp. tmpfile, err := ioutil.TempFile(runArgs.InputDir, "plan-for-import-*.tfplan") if err != nil { - return false, fmt.Errorf("create temp file: %v", err) + return nil, fmt.Errorf("create temp file: %v", err) } defer os.Remove(tmpfile.Name()) planName := path.Base(tmpfile.Name()) if out, err := tfCmdOutput("plan", "-out", planName); err != nil { - return false, fmt.Errorf("plan: %v\n%v", err, string(out)) + return nil, fmt.Errorf("plan: %v\n%v", err, string(out)) } b, err := tfCmdOutput("show", "-json", planName) if err != nil { - return false, fmt.Errorf("show: %v\n%v", err, string(b)) + return nil, fmt.Errorf("show: %v\n%v", err, string(b)) } // Load only "create" changes. createChanges, err := terraform.ReadPlanChanges(b, []string{"create"}) if err != nil { - return false, fmt.Errorf("read Terraform plan changes: %q", err) + return nil, fmt.Errorf("read Terraform plan changes: %q", err) } // Import all importable create changes. - importedSomething := false - var errs []string var importCmds []string var notImportableMsgs []string + var failures []string for _, cc := range createChanges { // If previously skipped, skip again if _, ok := skipped[cc.Address]; ok { @@ -610,7 +676,7 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str // This is needed to determine if it's possible to import the resource. pcv, err := terraform.ReadProviderConfigValues(b, cc.Kind, cc.Name) if err != nil { - return false, fmt.Errorf("read provider config values from the Terraform plan: %q", err) + return nil, fmt.Errorf("read provider config values from the Terraform plan: %q", err) } // Try to convert to an importable resource. @@ -656,28 +722,29 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str } // Handle the different outcomes of the import attempt. - var ie *importer.InsufficientInfoErr + var iie *importer.InsufficientInfoErr var se *importer.SkipErr switch { // err will only be nil when the import succeed. // Import succeeded, print the success output. case err == nil: // Import succeeded. - importedSomething = true + log.Printf("Successfully imported %v\n", cc.Address) + successes = append(successes, cc.Address) // Check if the user manually skipped the import. case errors.As(err, &se): skipped[cc.Address] = true // Check if the error indicates insufficient information. - case errors.As(err, &ie): - log.Printf("Insufficient information to import %q\n", cc.Address) + case errors.As(err, &iie): + log.Printf("Insufficient information to import %q, missing fields %s\n", cc.Address, strings.Join(iie.MissingFields, ",")) - errMsg := fmt.Sprintf("%q (insufficient information)", cc.Address) + msg := fmt.Sprintf("%v (insufficient information)", cc.Address) if runArgs.Verbose { - errMsg = fmt.Sprintf("%q ; insufficient information; full error: %v", cc.Address, err) + msg = fmt.Sprintf("%v ; insufficient information; full error: %v", cc.Address, err) } - errs = append(errs, errMsg) + failures = append(failures, msg) // Check if error indicates resource is not importable or does not exist. // err will be `exit code 1` even when it failed because the resource is not importable or already exists. @@ -690,11 +757,11 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str default: log.Printf("Failed to import %q\n", cc.Address) - errMsg := fmt.Sprintf("%q (error while running terraform import)", cc.Address) + msg := fmt.Sprintf("%v (error while running terraform import)", cc.Address) if runArgs.Verbose { - errMsg = fmt.Sprintf("%q ; full error: %v\n%v", cc.Address, err, output) + msg = fmt.Sprintf("%v ; full error: %v\n%v", cc.Address, err, output) } - errs = append(errs, errMsg) + failures = append(failures, msg) } } @@ -702,7 +769,7 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str if len(importCmds) > 0 { log.Printf("Import commands:") fmt.Printf("cd %v\n", runArgs.InputDir) - fmt.Printf("%v\n", strings.Join(importCmds, "\n")) + fmt.Printf("%v\n", strings.Join(importCmds, "")) } if len(notImportableMsgs) > 0 { @@ -710,27 +777,12 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str fmt.Printf("%v\n", strings.Join(notImportableMsgs, "")) } - return false, nil - } - - if importedSomething { - // Time to retry. Some resources imported successfully, but others didn't. - return true, nil - } - log.Printf("No resources imported.") - - if len(skipped) > 0 { - // Don't treat manual skips as errors. - var s []string - for resource := range skipped { - s = append(s, resource) - } - log.Printf("Skipped %d resources:\n%v", len(skipped), strings.Join(s, "\n")) + return nil, nil } - if len(errs) > 0 { - return false, fmt.Errorf("failed to find or import %v resources. Use --verbose flag to see detailed error messages.\n%v", len(errs), strings.Join(errs, "\n")) + if len(failures) > 0 { + return successes, &importError{failures} } - return false, nil + return successes, nil }