diff --git a/cmd/report_test.go b/cmd/report_test.go index 67580ab1..c788a899 100644 --- a/cmd/report_test.go +++ b/cmd/report_test.go @@ -199,6 +199,17 @@ func TestReport(t *testing.T) { }, wantErr: false, }, + { + name: "Should pass writing output to junit file", + args: []string{ + "-w", + "-o", + "junit", + string(apireportsummary.AnnotationsSummary), + "test/report.xml", + }, + wantErr: false, + }, { name: "Should pass for skip digest check", args: []string{ diff --git a/cmd/verify.go b/cmd/verify.go index 6c152f2e..b99e01b2 100644 --- a/cmd/verify.go +++ b/cmd/verify.go @@ -51,6 +51,8 @@ var ( disabledChecksFlag []string // outputFormatFlag contains the output format the user has specified: default, yaml or json. outputFormatFlag string + // junitFileFlag set the output file to output results in junit format. + junitFileFlag string // openshiftVersionFlag set the value of `certifiedOpenShiftVersions` in the report openshiftVersionFlag string // write report to file @@ -143,10 +145,9 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command { reportName := "" if reportToFile { + reportName = "report.yaml" if outputFormatFlag == "json" { reportName = "report.json" - } else { - reportName = "report.yaml" } } @@ -215,6 +216,14 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command { return reportErr } + if junitFileFlag != "" { + junitReport, junitReportErr := verifier.GetReport().JUnitContent() + if junitReportErr != nil { + utils.LogWarning(fmt.Sprintf("Failed to write JUnit output: %s", junitReportErr)) + } + utils.WriteToFile(junitReport, junitFileFlag) + } + utils.WriteStdOut(report) utils.WriteLogs(outputFormatFlag) @@ -237,7 +246,9 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command { cmd.Flags().StringSliceVarP(&disabledChecksFlag, "disable", "x", nil, "all checks will be enabled except the informed ones") - cmd.Flags().StringVarP(&outputFormatFlag, "output", "o", "", "the output format: default, json or yaml") + cmd.Flags().StringVarP(&outputFormatFlag, "output", "o", "", "the output format: default, json, or yaml") + + cmd.Flags().StringVarP(&junitFileFlag, "write-junit-to", "j", "", "set the output file to output results in junit format") cmd.Flags().StringSliceVarP(&verifyOpts.Values, "set", "s", []string{}, "overrides a configuration, e.g: dummy.ok=false") diff --git a/internal/chartverifier/utils/logger.go b/internal/chartverifier/utils/logger.go index 8b7dfe36..5fb4bb70 100644 --- a/internal/chartverifier/utils/logger.go +++ b/internal/chartverifier/utils/logger.go @@ -109,6 +109,16 @@ func WriteStdOut(output string) { } } +func WriteToFile(output string, filename string) { + fileWriteSuccess := false + if len(filename) > 0 { + fileWriteSuccess = writeToFile(output, filename) + } + if !fileWriteSuccess { + LogError(fmt.Sprintf("Failed writing output to: %s", filename)) + } +} + func writeToStdOut(output string) { savedOut := cmd.OutOrStdout() cmd.SetOut(CmdStdout) diff --git a/pkg/chartverifier/report/report.go b/pkg/chartverifier/report/report.go index 93dbb0d9..cb4c3564 100644 --- a/pkg/chartverifier/report/report.go +++ b/pkg/chartverifier/report/report.go @@ -2,6 +2,7 @@ package report import ( "encoding/json" + "encoding/xml" "errors" "fmt" "io" @@ -20,8 +21,9 @@ const ( SkippedOutcomeType OutcomeType = "SKIPPED" UnknownOutcomeType OutcomeType = "UNKNOWN" - JSONReport ReportFormat = "json" - YamlReport ReportFormat = "yaml" + JSONReport ReportFormat = "json" + JUnitReport ReportFormat = "junit" + YamlReport ReportFormat = "yaml" ReportShaVersion string = "v1.9.0" ) @@ -45,6 +47,87 @@ func (r *Report) Init() APIReport { return r } +func (r *Report) JUnitContent() (string, error) { + junitContent := "" + + var passedTests []*CheckReport + var failedTests []*CheckReport + var skippedTests []*CheckReport + var unknownTests []*CheckReport + + for _, element := range r.Results { + switch element.Outcome { + case PassOutcomeType: + passedTests = append(passedTests, element) + case FailOutcomeType: + failedTests = append(failedTests, element) + case SkippedOutcomeType: + skippedTests = append(skippedTests, element) + case UnknownOutcomeType: + unknownTests = append(unknownTests, element) + } + } + + suites := JUnitTestSuites{} + testsuite := JUnitTestSuite{ + Tests: len(passedTests) + len(failedTests) + len(skippedTests) + len(unknownTests), + Failures: len(failedTests), + Skipped: len(skippedTests), + Name: "Red Hat Chart Verifier", + Properties: []JUnitProperty{}, + TestCases: []JUnitTestCase{}, + } + + for _, result := range passedTests { + testCase := JUnitTestCase{ + Classname: string(result.Type), + Name: string(result.Check), + Failure: nil, + Message: result.Reason, + } + testsuite.TestCases = append(testsuite.TestCases, testCase) + } + + for _, result := range append(failedTests, unknownTests...) { + testCase := JUnitTestCase{ + Classname: string(result.Type), + Name: string(result.Check), + Failure: &JUnitMessage{ + Message: string(result.Outcome), + Type: "", + Contents: result.Reason, + }, + } + testsuite.TestCases = append(testsuite.TestCases, testCase) + } + + for _, result := range skippedTests { + testCase := JUnitTestCase{ + Classname: string(result.Type), + Name: string(result.Check), + SkipMessage: &JUnitSkipMessage{ + Message: fmt.Sprintf("Skipped: %s", result.Reason), + }, + } + testsuite.TestCases = append(testsuite.TestCases, testCase) + } + + suites.Suites = append(suites.Suites, testsuite) + + bytes, err := xml.MarshalIndent(suites, "", "\t") + if err != nil { + o := fmt.Errorf("error formatting results with formatter %s: %v", + "junitxml", + err, + ) + + return "", o + } + junitContent = xml.Header + string(bytes) + + return junitContent, nil +} + func (r *Report) GetContent(format ReportFormat) (string, error) { reportContent := "" @@ -56,7 +139,7 @@ func (r *Report) GetContent(format ReportFormat) (string, error) { if format == JSONReport { b, marshalErr := json.Marshal(report) if marshalErr != nil { - return "", fmt.Errorf("report json marshal failed : %v", marshalErr) + return "", fmt.Errorf("report xml marshal failed : %v", marshalErr) } reportContent = string(b) } else { @@ -113,6 +196,11 @@ func (r *Report) loadReport() error { if unMarshalErr != nil { return fmt.Errorf("report json ummarshal failed : %v", unMarshalErr) } + } else if strings.HasPrefix(strings.TrimSpace(reportString), "") { + unMarshalErr := xml.Unmarshal([]byte(reportString), r) + if unMarshalErr != nil { + return fmt.Errorf("report junit ummarshal failed : %v", unMarshalErr) + } } else { unMarshalErr := yaml.Unmarshal([]byte(reportString), r) if unMarshalErr != nil { diff --git a/pkg/chartverifier/report/types.go b/pkg/chartverifier/report/types.go index a8c593ef..da8ea8e3 100644 --- a/pkg/chartverifier/report/types.go +++ b/pkg/chartverifier/report/types.go @@ -1,6 +1,7 @@ package report import ( + "encoding/xml" "net/url" helmchart "helm.sh/helm/v3/pkg/chart" @@ -67,3 +68,44 @@ type reportOptions struct { //nolint: stylecheck // complains Url should be URL reportUrl *url.URL } + +type JUnitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []JUnitTestSuite `xml:"testsuite"` +} + +type JUnitTestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Skipped int `xml:"skipped,attr"` + Name string `xml:"name,attr"` + Properties []JUnitProperty `xml:"properties>property,omitempty"` + TestCases []JUnitTestCase `xml:"testcase"` +} + +type JUnitTestCase struct { + XMLName xml.Name `xml:"testcase"` + Classname string `xml:"classname,attr"` + Name string `xml:"name,attr"` + SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"` + Failure *JUnitMessage `xml:"failure,omitempty"` + Warning *JUnitMessage `xml:"warning,omitempty"` + SystemOut string `xml:"system-out,omitempty"` + Message string `xml:",chardata"` +} + +type JUnitSkipMessage struct { + Message string `xml:"message,attr"` +} + +type JUnitProperty struct { + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +type JUnitMessage struct { + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + Contents string `xml:",chardata"` +}