Skip to content

Commit

Permalink
Importer: Refactor and add final summary of results (#796)
Browse files Browse the repository at this point in the history 
* Importer: Refactor for more accuracy

Keep track of succeeded, skipped, and errored imports in the main loop, and report on them at the end of it.

* Rename "errs" to "failures"

* Use custom err and summary template

* Wording

* Address comments
  • Loading branch information
@MartinPetkov
MartinPetkov committed Feb 11, 2021
1 parent 062ac4c commit 9d4f63b
Showing 1 changed file with 95 additions and 43 deletions.
138 changes: 95 additions & 43 deletions internal/tfimport/tfimport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

"github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/fileutil"
"github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/runner"
"github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/template"
"github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/terraform"
"github.com/GoogleCloudPlatform/healthcare-data-protection-suite/internal/tfimport/importer"
)
Expand Down Expand Up @@ -517,6 +518,40 @@ func DoesNotExist(output string) bool {
return reDoesNotExist.FindStringIndex(output) != nil
}

// importError is an error indicating a resource failed to be imported.
// This doesn't indicate an error in running the steps around trying an import,
// it's specifically for terraform failing to import a resource for a reason
// that can't be handled (i.e. does not exist or not importable at all).
type importError struct {
// errMgs contains the error messages for all failed imports.
errMsgs []string
}

func (e *importError) Error() string {
return fmt.Sprintf("failed to find or import %v resources. Use --verbose flag to see detailed error messages.", len(e.errMsgs))
}

var summaryTmpl string = `
Summary:
Found {{.total}} importable resources
Successfully imported {{len .successes}}{{if gt (len .successes) 0}}:{{end}}
{{- range $resource := .successes}}
- {{$resource}}
{{- end}}
Skipped {{len .skipped}}{{if gt (len .skipped) 0}}:{{end}}
{{- range $resource := .skipped}}
- {{$resource}}
{{- end}}
Failed to import {{len .failures}}{{if gt (len .failures) 0}}:{{end}}
{{- range $resource := .failures}}
- {{$resource}}
{{- end}}
`

// RunArgs are the supported tfimport run arguments.
type RunArgs struct {
InputDir string
Expand All @@ -538,29 +573,61 @@ func Run(rn runner.Runner, importRn runner.Runner, runArgs *RunArgs) error {
return fmt.Errorf("expand path %q: %v", inputDir, err)
}

var successesTotal, failuresTotal []string
skipped := make(map[string]bool)
var ie *importError
for {
retry, err := planAndImport(rn, importRn, runArgs, skipped)
if err != nil {
failuresTotal = nil
successes, err := planAndImport(rn, importRn, runArgs, skipped)
successesTotal = append(successesTotal, successes...)

if err == nil {
// Either no successes, or no errors.
// No need to retry.
break
}

if !errors.As(err, &ie) {
// It's some other kind of error and we should return it.
return err
}

// If all imports succeeded, or none did, stop trying to import things, since we aren't making progress anymore.
if !retry {
// It's an importError.
failuresTotal = ie.errMsgs

// If no successes, break out of the loop.
if len(successes) <= 0 {
break
}

// Some resources imported successfully, but others didn't.
// Time to retry.
log.Println("Some imports succeeded but others did not. Retrying the import, in case dependent values have now been populated.")
}

buf, err := template.WriteBuffer(summaryTmpl, map[string]interface{}{
"total": len(successesTotal) + len(skipped) + len(failuresTotal),
"successes": successesTotal,
"skipped": skipped,
"failures": failuresTotal,
})
if err != nil {
return fmt.Errorf("building summary template: %v", err)
}
log.Printf("%s\n", buf)

if ie != nil {
return ie
}

return nil
}

// This function does the full plan and import cycle.
// If it imported some resources but failed to import others, it will return true for retry. This is a simple way to solve dependencies without having to figure out the graph.
// A specific case: GKE node pool name depends on random_id; import the random_id first, then do the cycle again and import the node pool.
// skipped is a map to be filled with skipped resources so they are skipped on subsequent runs too.
func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[string]bool) (retry bool, err error) {
func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[string]bool) (successes []string, err error) {
// Create Terraform command runners.
tfCmdOutput := func(args ...string) ([]byte, error) {
cmd := exec.Command(runArgs.TerraformPath, args...)
Expand All @@ -570,36 +637,35 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str

// Init is safe to run on an already-initialized config dir.
if out, err := tfCmdOutput("init"); err != nil {
return false, fmt.Errorf("init: %v\v%v", err, string(out))
return nil, fmt.Errorf("init: %v\v%v", err, string(out))
}

// Generate and load the plan using a temp file.
// Use the .tf files input dir in case the system can't write to /tmp.
tmpfile, err := ioutil.TempFile(runArgs.InputDir, "plan-for-import-*.tfplan")
if err != nil {
return false, fmt.Errorf("create temp file: %v", err)
return nil, fmt.Errorf("create temp file: %v", err)
}
defer os.Remove(tmpfile.Name())
planName := path.Base(tmpfile.Name())
if out, err := tfCmdOutput("plan", "-out", planName); err != nil {
return false, fmt.Errorf("plan: %v\n%v", err, string(out))
return nil, fmt.Errorf("plan: %v\n%v", err, string(out))
}
b, err := tfCmdOutput("show", "-json", planName)
if err != nil {
return false, fmt.Errorf("show: %v\n%v", err, string(b))
return nil, fmt.Errorf("show: %v\n%v", err, string(b))
}

// Load only "create" changes.
createChanges, err := terraform.ReadPlanChanges(b, []string{"create"})
if err != nil {
return false, fmt.Errorf("read Terraform plan changes: %q", err)
return nil, fmt.Errorf("read Terraform plan changes: %q", err)
}

// Import all importable create changes.
importedSomething := false
var errs []string
var importCmds []string
var notImportableMsgs []string
var failures []string
for _, cc := range createChanges {
// If previously skipped, skip again
if _, ok := skipped[cc.Address]; ok {
Expand All @@ -610,7 +676,7 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str
// This is needed to determine if it's possible to import the resource.
pcv, err := terraform.ReadProviderConfigValues(b, cc.Kind, cc.Name)
if err != nil {
return false, fmt.Errorf("read provider config values from the Terraform plan: %q", err)
return nil, fmt.Errorf("read provider config values from the Terraform plan: %q", err)
}

// Try to convert to an importable resource.
Expand Down Expand Up @@ -656,28 +722,29 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str
}

// Handle the different outcomes of the import attempt.
var ie *importer.InsufficientInfoErr
var iie *importer.InsufficientInfoErr
var se *importer.SkipErr
switch {
// err will only be nil when the import succeed.
// Import succeeded, print the success output.
case err == nil:
// Import succeeded.
importedSomething = true
log.Printf("Successfully imported %v\n", cc.Address)
successes = append(successes, cc.Address)

// Check if the user manually skipped the import.
case errors.As(err, &se):
skipped[cc.Address] = true

// Check if the error indicates insufficient information.
case errors.As(err, &ie):
log.Printf("Insufficient information to import %q\n", cc.Address)
case errors.As(err, &iie):
log.Printf("Insufficient information to import %q, missing fields %s\n", cc.Address, strings.Join(iie.MissingFields, ","))

errMsg := fmt.Sprintf("%q (insufficient information)", cc.Address)
msg := fmt.Sprintf("%v (insufficient information)", cc.Address)
if runArgs.Verbose {
errMsg = fmt.Sprintf("%q ; insufficient information; full error: %v", cc.Address, err)
msg = fmt.Sprintf("%v ; insufficient information; full error: %v", cc.Address, err)
}
errs = append(errs, errMsg)
failures = append(failures, msg)

// Check if error indicates resource is not importable or does not exist.
// err will be `exit code 1` even when it failed because the resource is not importable or already exists.
Expand All @@ -690,47 +757,32 @@ func planAndImport(rn, importRn runner.Runner, runArgs *RunArgs, skipped map[str
default:
log.Printf("Failed to import %q\n", cc.Address)

errMsg := fmt.Sprintf("%q (error while running terraform import)", cc.Address)
msg := fmt.Sprintf("%v (error while running terraform import)", cc.Address)
if runArgs.Verbose {
errMsg = fmt.Sprintf("%q ; full error: %v\n%v", cc.Address, err, output)
msg = fmt.Sprintf("%v ; full error: %v\n%v", cc.Address, err, output)
}
errs = append(errs, errMsg)
failures = append(failures, msg)
}
}

if runArgs.DryRun {
if len(importCmds) > 0 {
log.Printf("Import commands:")
fmt.Printf("cd %v\n", runArgs.InputDir)
fmt.Printf("%v\n", strings.Join(importCmds, "\n"))
fmt.Printf("%v\n", strings.Join(importCmds, ""))
}

if len(notImportableMsgs) > 0 {
log.Printf("The following resources are NOT importable:")
fmt.Printf("%v\n", strings.Join(notImportableMsgs, ""))
}

return false, nil
}

if importedSomething {
// Time to retry. Some resources imported successfully, but others didn't.
return true, nil
}
log.Printf("No resources imported.")

if len(skipped) > 0 {
// Don't treat manual skips as errors.
var s []string
for resource := range skipped {
s = append(s, resource)
}
log.Printf("Skipped %d resources:\n%v", len(skipped), strings.Join(s, "\n"))
return nil, nil
}

if len(errs) > 0 {
return false, fmt.Errorf("failed to find or import %v resources. Use --verbose flag to see detailed error messages.\n%v", len(errs), strings.Join(errs, "\n"))
if len(failures) > 0 {
return successes, &importError{failures}
}

return false, nil
return successes, nil
}

0 comments on commit 9d4f63b

Please sign in to comment.