Adding JUnit support to chart verifier report

cmd/report_test.go

@@ -199,6 +199,17 @@ func TestReport(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "Should pass writing output to junit file",
+			args: []string{
+				"-w",
+				"-o",
+				"junit",
+				string(apireportsummary.AnnotationsSummary),
+				"test/report.xml",
+			},
+			wantErr: false,
+		},
 		{
 			name: "Should pass for skip digest check",
 			args: []string{

cmd/verify.go

@@ -51,6 +51,8 @@ var (
 	disabledChecksFlag []string
 	// outputFormatFlag contains the output format the user has specified: default, yaml or json.
 	outputFormatFlag string
+	// junitFileFlag set the output file to output results in junit format.
+	junitFileFlag string
 	// openshiftVersionFlag set the value of `certifiedOpenShiftVersions` in the report
 	openshiftVersionFlag string
 	// write report to file
@@ -143,10 +145,9 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command {
 
 			reportName := ""
 			if reportToFile {
+				reportName = "report.yaml"
 				if outputFormatFlag == "json" {
 					reportName = "report.json"
-				} else {
-					reportName = "report.yaml"
 				}
 			}
 
@@ -215,6 +216,14 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command {
 				return reportErr
 			}
 
+			if junitFileFlag != "" {
+				junitReport, junitReportErr := verifier.GetReport().JUnitContent()
+				if junitReportErr != nil {
+					utils.LogWarning(fmt.Sprintf("Failed to write JUnit output: %s", junitReportErr))
+				}
+				utils.WriteToFile(junitReport, junitFileFlag)
+			}
+
 			utils.WriteStdOut(report)
 
 			utils.WriteLogs(outputFormatFlag)
@@ -237,7 +246,9 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command {
 
 	cmd.Flags().StringSliceVarP(&disabledChecksFlag, "disable", "x", nil, "all checks will be enabled except the informed ones")
 
-	cmd.Flags().StringVarP(&outputFormatFlag, "output", "o", "", "the output format: default, json or yaml")
+	cmd.Flags().StringVarP(&outputFormatFlag, "output", "o", "", "the output format: default, json, or yaml")
+
+	cmd.Flags().StringVarP(&junitFileFlag, "write-junit-to", "j", "", "set the output file to output results in junit format")
 
 	cmd.Flags().StringSliceVarP(&verifyOpts.Values, "set", "s", []string{}, "overrides a configuration, e.g: dummy.ok=false")
 

internal/chartverifier/utils/logger.go

@@ -109,6 +109,16 @@ func WriteStdOut(output string) {
 	}
 }
 
+func WriteToFile(output string, filename string) {
+	fileWriteSuccess := false
+	if len(filename) > 0 {
+		fileWriteSuccess = writeToFile(output, filename)
+	}
+	if !fileWriteSuccess {
+		LogError(fmt.Sprintf("Failed writing output to: %s", filename))
+	}
+}
+
 func writeToStdOut(output string) {
 	savedOut := cmd.OutOrStdout()
 	cmd.SetOut(CmdStdout)

pkg/chartverifier/report/report.go

@@ -2,6 +2,7 @@ package report
 
 import (
 	"encoding/json"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
@@ -20,8 +21,9 @@ const (
 	SkippedOutcomeType OutcomeType = "SKIPPED"
 	UnknownOutcomeType OutcomeType = "UNKNOWN"
 
-	JSONReport ReportFormat = "json"
-	YamlReport ReportFormat = "yaml"
+	JSONReport  ReportFormat = "json"
+	JUnitReport ReportFormat = "junit"
+	YamlReport  ReportFormat = "yaml"
 
 	ReportShaVersion string = "v1.9.0"
 )
@@ -45,6 +47,87 @@ func (r *Report) Init() APIReport {
 	return r
 }
 
+func (r *Report) JUnitContent() (string, error) {
+	junitContent := ""
+
+	var passedTests []*CheckReport
+	var failedTests []*CheckReport
+	var skippedTests []*CheckReport
+	var unknownTests []*CheckReport
+
+	for _, element := range r.Results {
+		switch element.Outcome {
+		case PassOutcomeType:
+			passedTests = append(passedTests, element)
+		case FailOutcomeType:
+			failedTests = append(failedTests, element)
+		case SkippedOutcomeType:
+			skippedTests = append(skippedTests, element)
+		case UnknownOutcomeType:
+			unknownTests = append(unknownTests, element)
+		}
+	}
+
+	suites := JUnitTestSuites{}
+	testsuite := JUnitTestSuite{
+		Tests:      len(passedTests) + len(failedTests) + len(skippedTests) + len(unknownTests),
+		Failures:   len(failedTests),
+		Skipped:    len(skippedTests),
+		Name:       "Red Hat Chart Verifier",
+		Properties: []JUnitProperty{},
+		TestCases:  []JUnitTestCase{},
+	}
+
+	for _, result := range passedTests {
+		testCase := JUnitTestCase{
+			Classname: string(result.Type),
+			Name:      string(result.Check),
+			Failure:   nil,
+			Message:   result.Reason,
+		}
+		testsuite.TestCases = append(testsuite.TestCases, testCase)
+	}
+
+	for _, result := range append(failedTests, unknownTests...) {
+		testCase := JUnitTestCase{
+			Classname: string(result.Type),
+			Name:      string(result.Check),
+			Failure: &JUnitMessage{
+				Message:  string(result.Outcome),
+				Type:     "",
+				Contents: result.Reason,
+			},
+		}
+		testsuite.TestCases = append(testsuite.TestCases, testCase)
+	}
+
+	for _, result := range skippedTests {
+		testCase := JUnitTestCase{
+			Classname: string(result.Type),
+			Name:      string(result.Check),
+			SkipMessage: &JUnitSkipMessage{
+				Message: fmt.Sprintf("Skipped: %s", result.Reason),
+			},
+		}
+		testsuite.TestCases = append(testsuite.TestCases, testCase)
+	}
+
+	suites.Suites = append(suites.Suites, testsuite)
+
+	bytes, err := xml.MarshalIndent(suites, "", "\t")
+	if err != nil {
+		o := fmt.Errorf("error formatting results with formatter %s: %v",
+			"junitxml",
+			err,
+		)
+
+		return "", o
+	}
+	junitContent = xml.Header + string(bytes)
+
+	return junitContent, nil
+}
+
 func (r *Report) GetContent(format ReportFormat) (string, error) {
 	reportContent := ""
 
@@ -56,7 +139,7 @@ func (r *Report) GetContent(format ReportFormat) (string, error) {
 	if format == JSONReport {
 		b, marshalErr := json.Marshal(report)
 		if marshalErr != nil {
-			return "", fmt.Errorf("report json marshal failed : %v", marshalErr)
+			return "", fmt.Errorf("report xml marshal failed : %v", marshalErr)
 		}
 		reportContent = string(b)
 	} else {
@@ -113,6 +196,11 @@ func (r *Report) loadReport() error {
 		if unMarshalErr != nil {
 			return fmt.Errorf("report json ummarshal failed : %v", unMarshalErr)
 		}
+	} else if strings.HasPrefix(strings.TrimSpace(reportString), "<?xml version=\"1.0\" encoding=\"UTF-8\"?>") {
+		unMarshalErr := xml.Unmarshal([]byte(reportString), r)
+		if unMarshalErr != nil {
+			return fmt.Errorf("report junit ummarshal failed : %v", unMarshalErr)
+		}
 	} else {
 		unMarshalErr := yaml.Unmarshal([]byte(reportString), r)
 		if unMarshalErr != nil {

pkg/chartverifier/report/types.go

@@ -1,6 +1,7 @@
 package report
 
 import (
+	"encoding/xml"
 	"net/url"
 
 	helmchart "helm.sh/helm/v3/pkg/chart"
@@ -67,3 +68,44 @@ type reportOptions struct {
 	//nolint: stylecheck // complains Url should be URL
 	reportUrl *url.URL
 }
+
+type JUnitTestSuites struct {
+	XMLName xml.Name         `xml:"testsuites"`
+	Suites  []JUnitTestSuite `xml:"testsuite"`
+}
+
+type JUnitTestSuite struct {
+	XMLName    xml.Name        `xml:"testsuite"`
+	Tests      int             `xml:"tests,attr"`
+	Failures   int             `xml:"failures,attr"`
+	Skipped    int             `xml:"skipped,attr"`
+	Name       string          `xml:"name,attr"`
+	Properties []JUnitProperty `xml:"properties>property,omitempty"`
+	TestCases  []JUnitTestCase `xml:"testcase"`
+}
+
+type JUnitTestCase struct {
+	XMLName     xml.Name          `xml:"testcase"`
+	Classname   string            `xml:"classname,attr"`
+	Name        string            `xml:"name,attr"`
+	SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
+	Failure     *JUnitMessage     `xml:"failure,omitempty"`
+	Warning     *JUnitMessage     `xml:"warning,omitempty"`
+	SystemOut   string            `xml:"system-out,omitempty"`
+	Message     string            `xml:",chardata"`
+}
+
+type JUnitSkipMessage struct {
+	Message string `xml:"message,attr"`
+}
+
+type JUnitProperty struct {
+	Name  string `xml:"name,attr"`
+	Value string `xml:"value,attr"`
+}
+
+type JUnitMessage struct {
+	Message  string `xml:"message,attr"`
+	Type     string `xml:"type,attr"`
+	Contents string `xml:",chardata"`
+}