From fec18ea045a0411e83a076485c14ae94c1f1ab93 Mon Sep 17 00:00:00 2001 From: Dan Curran Date: Wed, 1 May 2024 05:39:04 -0500 Subject: [PATCH] Moving JUnit support to secondary flag rather than main output. Using structs to reorganize results and remove unnecessary metadata from Junit output --- cmd/verify.go | 18 +- internal/chartverifier/utils/logger.go | 10 ++ pkg/chartverifier/report/junitConverter.go | 198 --------------------- pkg/chartverifier/report/report.go | 88 ++++++++- pkg/chartverifier/report/types.go | 42 +++++ 5 files changed, 146 insertions(+), 210 deletions(-) delete mode 100644 pkg/chartverifier/report/junitConverter.go diff --git a/cmd/verify.go b/cmd/verify.go index 1405c363..f78c42c4 100644 --- a/cmd/verify.go +++ b/cmd/verify.go @@ -51,6 +51,8 @@ var ( disabledChecksFlag []string // outputFormatFlag contains the output format the user has specified: default, yaml or json. outputFormatFlag string + // junitFileFlag set the output file to output results in junit format. + junitFileFlag string // TODO(komish): The description of this would imply that it's important. We generally // allow users to set overrides using the --set flag. The compiler says it's unused, so something doesn't // quite align. For now, we'll ignore this. @@ -146,8 +148,6 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command { reportFormat := apireport.YamlReport if outputFormatFlag == "json" { reportFormat = apireport.JSONReport - } else if outputFormatFlag == "junit" { - reportFormat = apireport.JUnitReport } reportName := "" @@ -155,8 +155,6 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command { reportName = "report.yaml" if outputFormatFlag == "json" { reportName = "report.json" - } else if outputFormatFlag == "junit" { - reportName = "report.xml" } } @@ -225,6 +223,14 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command { return reportErr } + if junitFileFlag != "" { + junitReport, junitReportErr := verifier.GetReport().JUnitContent() + if junitReportErr != nil { + utils.LogWarning(fmt.Sprintf("Failed to write JUnit output: %s", junitReportErr)) + } + utils.WriteToFile(junitReport, junitFileFlag) + } + utils.WriteStdOut(report) utils.WriteLogs(outputFormatFlag) @@ -247,7 +253,9 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command { cmd.Flags().StringSliceVarP(&disabledChecksFlag, "disable", "x", nil, "all checks will be enabled except the informed ones") - cmd.Flags().StringVarP(&outputFormatFlag, "output", "o", "", "the output format: default, json, junit or yaml") + cmd.Flags().StringVarP(&outputFormatFlag, "output", "o", "", "the output format: default, json, or yaml") + + cmd.Flags().StringVarP(&junitFileFlag, "write-junit-to", "j", "", "set the output file to output results in junit format") cmd.Flags().StringSliceVarP(&verifyOpts.Values, "set", "s", []string{}, "overrides a configuration, e.g: dummy.ok=false") diff --git a/internal/chartverifier/utils/logger.go b/internal/chartverifier/utils/logger.go index 052bae8b..445072a5 100644 --- a/internal/chartverifier/utils/logger.go +++ b/internal/chartverifier/utils/logger.go @@ -109,6 +109,16 @@ func WriteStdOut(output string) { } } +func WriteToFile(output string, filename string) { + fileWriteSuccess := false + if len(filename) > 0 { + fileWriteSuccess = writeToFile(output, filename) + } + if !fileWriteSuccess { + LogError(fmt.Sprintf("Failed writing output to: %s", filename)) + } +} + func writeToStdOut(output string) { savedOut := cmd.OutOrStdout() cmd.SetOut(CmdStdout) diff --git a/pkg/chartverifier/report/junitConverter.go b/pkg/chartverifier/report/junitConverter.go deleted file mode 100644 index 2f0ca4c2..00000000 --- a/pkg/chartverifier/report/junitConverter.go +++ /dev/null @@ -1,198 +0,0 @@ -package report - -import ( - "encoding/xml" - "fmt" - "reflect" - "time" -) - -func encodeTokenArray(e *xml.Encoder, tokens []xml.Token) error { - for _, t := range tokens { - err := e.EncodeToken(t) - if err != nil { - return err - } - } - return e.Flush() -} - -func encodeString(e *xml.Encoder, name string, value string) error { - start := xml.StartElement{Name: xml.Name{"", name}} - tokens := []xml.Token{start} - tokens = append(tokens, xml.CharData(value), xml.EndElement{start.Name}) - - return encodeTokenArray(e, tokens) -} - -func encodeStringArray(e *xml.Encoder, name string, values []string) error { - start := xml.StartElement{Name: xml.Name{"", name}} - tokens := []xml.Token{start} - - for _, value := range values { - t := xml.StartElement{Name: xml.Name{"", "value"}} - tokens = append(tokens, t, xml.CharData(value), xml.EndElement{t.Name}) - } - - tokens = append(tokens, xml.EndElement{start.Name}) - - return encodeTokenArray(e, tokens) -} - -func encodeStringMap(e *xml.Encoder, name string, values map[string]string) error { - start := xml.StartElement{Name: xml.Name{"", name}} - tokens := []xml.Token{start} - - for key, value := range values { - t := xml.StartElement{Name: xml.Name{"", key}} - tokens = append(tokens, t, xml.CharData(value), xml.EndElement{t.Name}) - } - - tokens = append(tokens, xml.EndElement{start.Name}) - - return encodeTokenArray(e, tokens) -} - -func encodeReportMetadata(e *xml.Encoder, m ReportMetadata, start xml.StartElement) { - // Start ReportMetadata and encode what can be done automatically - e.EncodeToken(start) - e.Encode(m.ToolMetadata) - encodeString(e, "Overrides", m.Overrides) - - // Work through parts of the ChartData which is a problem due to Annotations field - chartData := m.ChartData - chartDataStartToken := xml.StartElement{Name: xml.Name{"", "ChartData"}} - e.EncodeToken(chartDataStartToken) - - // Loop through helmchart.Metadata Fields and encode strings/bools - v := reflect.ValueOf(*chartData) - typeOfS := v.Type() - for i := 0; i < v.NumField(); i++ { - fieldType := fmt.Sprintf("%T", v.Field(i).Interface()) - - if fieldType == "string" || fieldType == "bool" { - encodeString(e, - typeOfS.Field(i).Name, - fmt.Sprintf("%v", v.Field(i).Interface())) - } - } - - // Loop through helmchart.Metadata Fields - encodeStringArray(e, "Sources", chartData.Sources) - encodeStringArray(e, "Keywords", chartData.Keywords) - encodeStringMap(e, "Annotations", chartData.Annotations) - - // Maintainers Start - maintainersStartToken := xml.StartElement{Name: xml.Name{"", "Maintainers"}} - e.EncodeToken(maintainersStartToken) - e.Encode(chartData.Maintainers) - e.EncodeToken(xml.EndElement{maintainersStartToken.Name}) - // Maintainers End - - // Dependencies Start - dependenciesStartToken := xml.StartElement{Name: xml.Name{"", "Dependencies"}} - e.EncodeToken(dependenciesStartToken) - e.Encode(chartData.Dependencies) - e.EncodeToken(xml.EndElement{dependenciesStartToken.Name}) - // Dependencies End - - e.EncodeToken(xml.EndElement{chartDataStartToken.Name}) - // ChartData End - - e.EncodeToken(xml.EndElement{start.Name}) - // Metadata End - - e.Flush() -} - -func encodeResults(e *xml.Encoder, r Report) error { - tokens := []xml.Token{} - - for _, testcase := range r.Results { - - // Put Check, Type, and Outcome into XML testcase element - t := xml.StartElement{Name: xml.Name{"", "testcase"}, - Attr: []xml.Attr{ - {xml.Name{"", "name"}, string(testcase.Check)}, - {xml.Name{"", "classname"}, string(testcase.Reason)}, - {xml.Name{"", "assertions"}, "1"}, - }, - } - - //Create an element for Reason - reasonToken := xml.StartElement{Name: xml.Name{"", "system-out"}} - - tokens = append(tokens, t, - reasonToken, xml.CharData(testcase.Reason), xml.EndElement{reasonToken.Name}, - xml.EndElement{t.Name}) - - } - - return encodeTokenArray(e, tokens) -} - -func (r Report) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - - numTests := len(r.Results) - numFailures := 0 - numSkipped := 0 - numPassed := 0 - timestamp := time.Now().Format(time.RFC3339) - - for _, element := range r.Results { - switch element.Outcome { - case "FAIL": - numFailures += 1 - case "SKIPPED": - numSkipped += 1 - case "PASS": - numPassed += 1 - } - } - - junitStart := xml.StartElement{Name: xml.Name{"", "testsuites"}, - Attr: []xml.Attr{ - {xml.Name{"", "name"}, "chart-verifier test run"}, - {xml.Name{"", "tests"}, fmt.Sprint(numTests)}, - {xml.Name{"", "failures"}, fmt.Sprint(numFailures)}, - {xml.Name{"", "skipped"}, fmt.Sprint(numSkipped)}, - {xml.Name{"", "timestamp"}, timestamp}, - }, - } - propertiesStart := xml.StartElement{Name: xml.Name{"", "properties"}} - propertyStart := xml.StartElement{Name: xml.Name{"", "property"}, - Attr: []xml.Attr{ - {xml.Name{"", "name"}, "config"}, - }, - } - - e.EncodeToken(junitStart) - e.EncodeToken(propertiesStart) - e.EncodeToken(propertyStart) - encodeString(e, "ApiVersion", r.Apiversion) - encodeString(e, "Kind", r.Kind) - e.Encode(r.options) - encodeReportMetadata(e, r.Metadata, xml.StartElement{Name: xml.Name{"", "Metadata"}}) - e.EncodeToken(xml.EndElement{propertyStart.Name}) - e.EncodeToken(xml.EndElement{propertiesStart.Name}) - - testSuiteStart := xml.StartElement{Name: xml.Name{"", "testsuite"}, - Attr: []xml.Attr{ - {xml.Name{"", "name"}, "chart-verifier test run"}, - {xml.Name{"", "tests"}, fmt.Sprint(numTests)}, - {xml.Name{"", "failures"}, fmt.Sprint(numFailures)}, - {xml.Name{"", "skipped"}, fmt.Sprint(numSkipped)}, - {xml.Name{"", "timestamp"}, timestamp}, - }, - } - e.EncodeToken(testSuiteStart) - - // e.Encode(r.Results) - encodeResults(e, r) - - // flush to ensure tokens are written - e.EncodeToken(xml.EndElement{testSuiteStart.Name}) - e.EncodeToken(xml.EndElement{junitStart.Name}) - - return e.Flush() -} diff --git a/pkg/chartverifier/report/report.go b/pkg/chartverifier/report/report.go index 96c39d8c..cb4c3564 100644 --- a/pkg/chartverifier/report/report.go +++ b/pkg/chartverifier/report/report.go @@ -47,6 +47,87 @@ func (r *Report) Init() APIReport { return r } +func (r *Report) JUnitContent() (string, error) { + junitContent := "" + + var passedTests []*CheckReport + var failedTests []*CheckReport + var skippedTests []*CheckReport + var unknownTests []*CheckReport + + for _, element := range r.Results { + switch element.Outcome { + case PassOutcomeType: + passedTests = append(passedTests, element) + case FailOutcomeType: + failedTests = append(failedTests, element) + case SkippedOutcomeType: + skippedTests = append(skippedTests, element) + case UnknownOutcomeType: + unknownTests = append(unknownTests, element) + } + } + + suites := JUnitTestSuites{} + testsuite := JUnitTestSuite{ + Tests: len(passedTests) + len(failedTests) + len(skippedTests) + len(unknownTests), + Failures: len(failedTests), + Skipped: len(skippedTests), + Name: "Red Hat Chart Verifier", + Properties: []JUnitProperty{}, + TestCases: []JUnitTestCase{}, + } + + for _, result := range passedTests { + testCase := JUnitTestCase{ + Classname: string(result.Type), + Name: string(result.Check), + Failure: nil, + Message: result.Reason, + } + testsuite.TestCases = append(testsuite.TestCases, testCase) + } + + for _, result := range append(failedTests, unknownTests...) { + testCase := JUnitTestCase{ + Classname: string(result.Type), + Name: string(result.Check), + Failure: &JUnitMessage{ + Message: string(result.Outcome), + Type: "", + Contents: result.Reason, + }, + } + testsuite.TestCases = append(testsuite.TestCases, testCase) + } + + for _, result := range skippedTests { + testCase := JUnitTestCase{ + Classname: string(result.Type), + Name: string(result.Check), + SkipMessage: &JUnitSkipMessage{ + Message: fmt.Sprintf("Skipped: %s", result.Reason), + }, + } + testsuite.TestCases = append(testsuite.TestCases, testCase) + } + + suites.Suites = append(suites.Suites, testsuite) + + bytes, err := xml.MarshalIndent(suites, "", "\t") + if err != nil { + o := fmt.Errorf("error formatting results with formatter %s: %v", + "junitxml", + err, + ) + + return "", o + } + junitContent = xml.Header + string(bytes) + + return junitContent, nil +} + func (r *Report) GetContent(format ReportFormat) (string, error) { reportContent := "" @@ -61,13 +142,6 @@ func (r *Report) GetContent(format ReportFormat) (string, error) { return "", fmt.Errorf("report xml marshal failed : %v", marshalErr) } reportContent = string(b) - } else if format == JUnitReport { - // convert report to JUnit - out, marshalErr := xml.MarshalIndent(r, " ", " ") - if marshalErr != nil { - return "", fmt.Errorf("report JUnit marshal failed: %v", marshalErr) - } - reportContent = xml.Header + string(out) } else { b, marshalErr := yaml.Marshal(report) if marshalErr != nil { diff --git a/pkg/chartverifier/report/types.go b/pkg/chartverifier/report/types.go index a8c593ef..da8ea8e3 100644 --- a/pkg/chartverifier/report/types.go +++ b/pkg/chartverifier/report/types.go @@ -1,6 +1,7 @@ package report import ( + "encoding/xml" "net/url" helmchart "helm.sh/helm/v3/pkg/chart" @@ -67,3 +68,44 @@ type reportOptions struct { //nolint: stylecheck // complains Url should be URL reportUrl *url.URL } + +type JUnitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []JUnitTestSuite `xml:"testsuite"` +} + +type JUnitTestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Skipped int `xml:"skipped,attr"` + Name string `xml:"name,attr"` + Properties []JUnitProperty `xml:"properties>property,omitempty"` + TestCases []JUnitTestCase `xml:"testcase"` +} + +type JUnitTestCase struct { + XMLName xml.Name `xml:"testcase"` + Classname string `xml:"classname,attr"` + Name string `xml:"name,attr"` + SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"` + Failure *JUnitMessage `xml:"failure,omitempty"` + Warning *JUnitMessage `xml:"warning,omitempty"` + SystemOut string `xml:"system-out,omitempty"` + Message string `xml:",chardata"` +} + +type JUnitSkipMessage struct { + Message string `xml:"message,attr"` +} + +type JUnitProperty struct { + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +type JUnitMessage struct { + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + Contents string `xml:",chardata"` +}