Skip to content

Commit 93a44f3

Browse files
stbenjamstbenjam
committed
feat(test): add OTE metadata and test source tracking to JUnit results
Extend JUnit test case schema with OpenShift Test Extensions (OTE) metadata attributes. Metadata fields added: - Lifecycle classification (blocking/informing) - Test execution timestamps (start-time, end-time) - Source tracking (source-image, source-binary, source-url, source-commit) The source fields allow tracking which container image, binary, repository, and commit produced each test result, enabling better traceability and debugging of test failures across different builds.
1 parent 5c72de0 commit 93a44f3

File tree

3 files changed

+313
-11
lines changed

3 files changed

+313
-11
lines changed

‎pkg/test/ginkgo/junit.go‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,40 @@ import (
1010
"time"
1111

1212
"github.com/openshift/origin/pkg/test"
13+
"github.com/openshift/origin/pkg/test/extensions"
1314
"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
1415

1516
"github.com/openshift/origin/pkg/version"
1617
)
1718

19+
// populateOTEMetadata adds OTE metadata attributes to a JUnit test case if available
20+
func populateOTEMetadata(testCase *junitapi.JUnitTestCase, extensionResult *extensions.ExtensionTestResult) {
21+
if extensionResult == nil {
22+
return
23+
}
24+
25+
// Test source information
26+
testCase.SourceImage = extensionResult.Source.SourceImage
27+
testCase.SourceBinary = extensionResult.Source.SourceBinary
28+
if extensionResult.Source.Source != nil {
29+
testCase.SourceURL = extensionResult.Source.SourceURL
30+
testCase.SourceCommit = extensionResult.Source.Commit
31+
}
32+
33+
// Set lifecycle attribute
34+
testCase.Lifecycle = string(extensionResult.Lifecycle)
35+
36+
// Set start time attribute if available
37+
if extensionResult.StartTime != nil {
38+
testCase.StartTime = time.Time(*extensionResult.StartTime).UTC().Format(time.RFC3339)
39+
}
40+
41+
// Set end time attribute if available
42+
if extensionResult.EndTime != nil {
43+
testCase.EndTime = time.Time(*extensionResult.EndTime).UTC().Format(time.RFC3339)
44+
}
45+
}
46+
1847
func generateJUnitTestSuiteResults(
1948
name string,
2049
duration time.Duration,
@@ -36,49 +65,59 @@ func generateJUnitTestSuiteResults(
3665
case test.skipped:
3766
s.NumTests++
3867
s.NumSkipped++
39-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
68+
testCase := &junitapi.JUnitTestCase{
4069
Name: test.name,
4170
SystemOut: string(test.testOutputBytes),
4271
Duration: test.duration.Seconds(),
4372
SkipMessage: &junitapi.SkipMessage{
4473
Message: lastLinesUntil(string(test.testOutputBytes), 100, "skip ["),
4574
},
46-
})
75+
}
76+
populateOTEMetadata(testCase, test.extensionTestResult)
77+
s.TestCases = append(s.TestCases, testCase)
4778
case test.failed:
4879
s.NumTests++
4980
s.NumFailed++
50-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
81+
testCase := &junitapi.JUnitTestCase{
5182
Name: test.name,
5283
SystemOut: string(test.testOutputBytes),
5384
Duration: test.duration.Seconds(),
5485
FailureOutput: &junitapi.FailureOutput{
5586
Output: lastLinesUntil(string(test.testOutputBytes), 100, "fail ["),
5687
},
57-
})
88+
}
89+
populateOTEMetadata(testCase, test.extensionTestResult)
90+
s.TestCases = append(s.TestCases, testCase)
5891
case test.flake:
5992
s.NumTests++
6093
s.NumFailed++
61-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
94+
failedTestCase := &junitapi.JUnitTestCase{
6295
Name: test.name,
6396
SystemOut: string(test.testOutputBytes),
6497
Duration: test.duration.Seconds(),
6598
FailureOutput: &junitapi.FailureOutput{
6699
Output: lastLinesUntil(string(test.testOutputBytes), 100, "flake:"),
67100
},
68-
})
101+
}
102+
populateOTEMetadata(failedTestCase, test.extensionTestResult)
103+
s.TestCases = append(s.TestCases, failedTestCase)
69104

70105
// also add the successful junit result:
71106
s.NumTests++
72-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
107+
successTestCase := &junitapi.JUnitTestCase{
73108
Name: test.name,
74109
Duration: test.duration.Seconds(),
75-
})
110+
}
111+
populateOTEMetadata(successTestCase, test.extensionTestResult)
112+
s.TestCases = append(s.TestCases, successTestCase)
76113
case test.success:
77114
s.NumTests++
78-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
115+
testCase := &junitapi.JUnitTestCase{
79116
Name: test.name,
80117
Duration: test.duration.Seconds(),
81-
})
118+
}
119+
populateOTEMetadata(testCase, test.extensionTestResult)
120+
s.TestCases = append(s.TestCases, testCase)
82121
}
83122
}
84123
for _, result := range syntheticTestResults {

‎pkg/test/ginkgo/junit_test.go‎

Lines changed: 255 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,17 @@
11
package ginkgo
22

3-
import "testing"
3+
import (
4+
"encoding/xml"
5+
"strings"
6+
"testing"
7+
"time"
8+
9+
"github.com/openshift-eng/openshift-tests-extension/pkg/dbtime"
10+
"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
11+
"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
12+
"github.com/openshift/origin/pkg/test/extensions"
13+
"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
14+
)
415

516
func Test_lastLines(t *testing.T) {
617
tests := []struct {
@@ -33,3 +44,246 @@ func Test_lastLines(t *testing.T) {
3344
})
3445
}
3546
}
47+
48+
func Test_populateOTEMetadata(t *testing.T) {
49+
startTime := time.Date(2023, 12, 25, 10, 0, 0, 0, time.UTC)
50+
endTime := time.Date(2023, 12, 25, 10, 5, 0, 0, time.UTC)
51+
52+
tests := []struct {
53+
name string
54+
extensionResult *extensions.ExtensionTestResult
55+
expectedLifecycle string
56+
expectedStartTime string
57+
expectedEndTime string
58+
expectedSourceImage string
59+
expectedSourceBinary string
60+
expectedSourceURL string
61+
expectedSourceCommit string
62+
}{
63+
{
64+
name: "nil extension result",
65+
extensionResult: nil,
66+
expectedLifecycle: "",
67+
expectedStartTime: "",
68+
expectedEndTime: "",
69+
expectedSourceImage: "",
70+
expectedSourceBinary: "",
71+
expectedSourceURL: "",
72+
expectedSourceCommit: "",
73+
},
74+
{
75+
name: "complete extension result",
76+
extensionResult: &extensions.ExtensionTestResult{
77+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
78+
Name: "test-case",
79+
Lifecycle: extensiontests.LifecycleBlocking,
80+
StartTime: dbtime.Ptr(startTime),
81+
EndTime: dbtime.Ptr(endTime),
82+
},
83+
Source: extensions.Source{
84+
Source: &extension.Source{
85+
Commit: "abc123def456",
86+
SourceURL: "https://github.com/example/repo",
87+
},
88+
SourceImage: "tests",
89+
SourceBinary: "openshift-tests",
90+
},
91+
},
92+
expectedLifecycle: "blocking",
93+
expectedStartTime: "2023-12-25T10:00:00Z",
94+
expectedEndTime: "2023-12-25T10:05:00Z",
95+
expectedSourceImage: "tests",
96+
expectedSourceBinary: "openshift-tests",
97+
expectedSourceURL: "https://github.com/example/repo",
98+
expectedSourceCommit: "abc123def456",
99+
},
100+
{
101+
name: "informing lifecycle",
102+
extensionResult: &extensions.ExtensionTestResult{
103+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
104+
Name: "test-case",
105+
Lifecycle: extensiontests.LifecycleInforming,
106+
StartTime: dbtime.Ptr(startTime),
107+
EndTime: dbtime.Ptr(endTime),
108+
},
109+
Source: extensions.Source{
110+
Source: &extension.Source{
111+
Commit: "xyz789",
112+
SourceURL: "https://github.com/openshift/origin",
113+
},
114+
SourceImage: "tests",
115+
SourceBinary: "openshift-tests",
116+
},
117+
},
118+
expectedLifecycle: "informing",
119+
expectedStartTime: "2023-12-25T10:00:00Z",
120+
expectedEndTime: "2023-12-25T10:05:00Z",
121+
expectedSourceImage: "tests",
122+
expectedSourceBinary: "openshift-tests",
123+
expectedSourceURL: "https://github.com/openshift/origin",
124+
expectedSourceCommit: "xyz789",
125+
},
126+
{
127+
name: "missing time fields",
128+
extensionResult: &extensions.ExtensionTestResult{
129+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
130+
Name: "test-case",
131+
Lifecycle: extensiontests.LifecycleBlocking,
132+
StartTime: nil,
133+
EndTime: nil,
134+
},
135+
},
136+
expectedLifecycle: "blocking",
137+
expectedStartTime: "",
138+
expectedEndTime: "",
139+
expectedSourceImage: "",
140+
expectedSourceBinary: "",
141+
expectedSourceURL: "",
142+
expectedSourceCommit: "",
143+
},
144+
{
145+
name: "partial source information",
146+
extensionResult: &extensions.ExtensionTestResult{
147+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
148+
Name: "test-case",
149+
Lifecycle: extensiontests.LifecycleBlocking,
150+
StartTime: dbtime.Ptr(startTime),
151+
EndTime: dbtime.Ptr(endTime),
152+
},
153+
Source: extensions.Source{
154+
Source: &extension.Source{
155+
Commit: "abc123",
156+
},
157+
SourceImage: "tests",
158+
},
159+
},
160+
expectedLifecycle: "blocking",
161+
expectedStartTime: "2023-12-25T10:00:00Z",
162+
expectedEndTime: "2023-12-25T10:05:00Z",
163+
expectedSourceImage: "tests",
164+
expectedSourceBinary: "",
165+
expectedSourceURL: "",
166+
expectedSourceCommit: "abc123",
167+
},
168+
}
169+
170+
for _, tt := range tests {
171+
t.Run(tt.name, func(t *testing.T) {
172+
testCase := &junitapi.JUnitTestCase{
173+
Name: "test-case",
174+
Duration: 300.0, // 5 minutes
175+
}
176+
177+
populateOTEMetadata(testCase, tt.extensionResult)
178+
179+
if testCase.Lifecycle != tt.expectedLifecycle {
180+
t.Errorf("Lifecycle = %q, want %q", testCase.Lifecycle, tt.expectedLifecycle)
181+
}
182+
if testCase.StartTime != tt.expectedStartTime {
183+
t.Errorf("StartTime = %q, want %q", testCase.StartTime, tt.expectedStartTime)
184+
}
185+
if testCase.EndTime != tt.expectedEndTime {
186+
t.Errorf("EndTime = %q, want %q", testCase.EndTime, tt.expectedEndTime)
187+
}
188+
if testCase.SourceImage != tt.expectedSourceImage {
189+
t.Errorf("SourceImage = %q, want %q", testCase.SourceImage, tt.expectedSourceImage)
190+
}
191+
if testCase.SourceBinary != tt.expectedSourceBinary {
192+
t.Errorf("SourceBinary = %q, want %q", testCase.SourceBinary, tt.expectedSourceBinary)
193+
}
194+
if testCase.SourceURL != tt.expectedSourceURL {
195+
t.Errorf("SourceURL = %q, want %q", testCase.SourceURL, tt.expectedSourceURL)
196+
}
197+
if testCase.SourceCommit != tt.expectedSourceCommit {
198+
t.Errorf("SourceCommit = %q, want %q", testCase.SourceCommit, tt.expectedSourceCommit)
199+
}
200+
})
201+
}
202+
}
203+
204+
func Test_junitXMLWithOTEAttributes(t *testing.T) {
205+
startTime := time.Date(2023, 12, 25, 10, 0, 0, 0, time.UTC)
206+
endTime := time.Date(2023, 12, 25, 10, 5, 0, 0, time.UTC)
207+
208+
// Create a JUnit test case and populate it with OTE metadata
209+
extensionResult := &extensions.ExtensionTestResult{
210+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
211+
Name: "example-test",
212+
Lifecycle: extensiontests.LifecycleBlocking,
213+
StartTime: dbtime.Ptr(startTime),
214+
EndTime: dbtime.Ptr(endTime),
215+
},
216+
Source: extensions.Source{
217+
Source: &extension.Source{
218+
Commit: "abc123def456789",
219+
SourceURL: "https://github.com/openshift/origin",
220+
},
221+
SourceImage: "tests",
222+
SourceBinary: "openshift-tests",
223+
},
224+
}
225+
226+
junitTestCase := &junitapi.JUnitTestCase{
227+
Name: "example-test",
228+
Duration: 300.0, // 5 minutes
229+
}
230+
231+
// Populate the OTE metadata
232+
populateOTEMetadata(junitTestCase, extensionResult)
233+
234+
// Create a test suite containing our test case
235+
suite := &junitapi.JUnitTestSuite{
236+
Name: "test-suite",
237+
NumTests: 1,
238+
Duration: 300.0,
239+
TestCases: []*junitapi.JUnitTestCase{junitTestCase},
240+
}
241+
242+
// Verify OTE metadata is present
243+
if junitTestCase.Lifecycle != "blocking" {
244+
t.Errorf("Lifecycle = %q, want %q", junitTestCase.Lifecycle, "blocking")
245+
}
246+
if junitTestCase.StartTime != "2023-12-25T10:00:00Z" {
247+
t.Errorf("StartTime = %q, want %q", junitTestCase.StartTime, "2023-12-25T10:00:00Z")
248+
}
249+
if junitTestCase.EndTime != "2023-12-25T10:05:00Z" {
250+
t.Errorf("EndTime = %q, want %q", junitTestCase.EndTime, "2023-12-25T10:05:00Z")
251+
}
252+
if junitTestCase.SourceImage != "tests" {
253+
t.Errorf("SourceImage = %q, want %q", junitTestCase.SourceImage, "tests")
254+
}
255+
if junitTestCase.SourceBinary != "openshift-tests" {
256+
t.Errorf("SourceBinary = %q, want %q", junitTestCase.SourceBinary, "openshift-tests")
257+
}
258+
if junitTestCase.SourceURL != "https://github.com/openshift/origin" {
259+
t.Errorf("SourceURL = %q, want %q", junitTestCase.SourceURL, "https://github.com/openshift/origin")
260+
}
261+
if junitTestCase.SourceCommit != "abc123def456789" {
262+
t.Errorf("SourceCommit = %q, want %q", junitTestCase.SourceCommit, "abc123def456789")
263+
}
264+
265+
// Verify XML marshaling includes the new attributes
266+
xmlData, err := xml.MarshalIndent(suite, "", " ")
267+
if err != nil {
268+
t.Fatalf("Failed to marshal XML: %v", err)
269+
}
270+
271+
xmlString := string(xmlData)
272+
273+
// Check that our custom attributes are in the XML
274+
expectedAttributes := []string{
275+
`lifecycle="blocking"`,
276+
`start-time="2023-12-25T10:00:00Z"`,
277+
`end-time="2023-12-25T10:05:00Z"`,
278+
`source-image="tests"`,
279+
`source-binary="openshift-tests"`,
280+
`source-url="https://github.com/openshift/origin"`,
281+
`source-commit="abc123def456789"`,
282+
}
283+
284+
for _, attr := range expectedAttributes {
285+
if !strings.Contains(xmlString, attr) {
286+
t.Errorf("XML does not contain expected attribute: %s\nXML output:\n%s", attr, xmlString)
287+
}
288+
}
289+
}

‎pkg/test/ginkgo/junitapi/types.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@ type JUnitTestCase struct {
6666
// Duration is the time taken in seconds to run the test
6767
Duration float64 `xml:"time,attr"`
6868

69+
// OTE metadata attributes
70+
StartTime string `xml:"start-time,attr,omitempty"`
71+
EndTime string `xml:"end-time,attr,omitempty"`
72+
Lifecycle string `xml:"lifecycle,attr,omitempty"`
73+
SourceImage string `xml:"source-image,attr,omitempty"`
74+
SourceBinary string `xml:"source-binary,attr,omitempty"`
75+
SourceURL string `xml:"source-url,attr,omitempty"`
76+
SourceCommit string `xml:"source-commit,attr,omitempty"`
77+
6978
// SkipMessage holds the reason why the test was skipped
7079
SkipMessage *SkipMessage `xml:"skipped"`
7180

0 commit comments

Comments
 (0)