Skip to content

Commit

Permalink
Add support for @afterall in XML report
Browse files Browse the repository at this point in the history
Writing a simple test like
```
package com.library;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;

public class AlwaysFailTest {
    @test
    public void doNothingTest() {
        System.out.println("Hi there!");
    }

    @afterall
    public static void alwaysFail() {
        throw new RuntimeException("Always failing.");
    }
}
```

Produces an XML that seems to look like it succeeds.
```xml
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
  <testsuite name="com.library.AlwaysFailTest" timestamp="2024-10-10T21:19:11.522176Z" hostname="KHK9NLVQGN" tests="2" failures="0" errors="0" disabled="0" skipped="0" package="">
    <properties/>
    <testcase name="doNothingTest" classname="com.library.AlwaysFailTest" time="0.03">
      <system-out><![CDATA[Hi there!
]]></system-out>
    </testcase>
  </testsuite>
</testsuites>
```

Bazel itself correctly identifies it fails. The Junit4 reporter also included with Bazel natively correclty reports the failure in the XML output.

Augment the Junit listener to add support for static methods and add them to the JUnit output.

Doing so produces the following XML.
```xml
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
  <testsuite name="com.library.AlwaysFailTest" timestamp="2024-10-10T21:49:02.096648Z" hostname="KHK9NLVQGN" tests="3" failures="1" errors="0" disabled="0" skipped="0" package="">
    <properties/>
    <testcase name="com.library.AlwaysFailTest" classname="com.library.AlwaysFailTest" time="0.05">
      <failure message="Always failing." type="java.lang.RuntimeException"><![CDATA[java.lang.RuntimeException: Always failing.
	at com.library.AlwaysFailTest.alwaysFail(AlwaysFailTest.java:20)
...
	at com.github.bazel_contrib.contrib_rules_jvm.junit5.JUnit5Runner.main(JUnit5Runner.java:39)
]]></failure>
    </testcase>
    <testcase name="doNothingTest" classname="com.library.AlwaysFailTest" time="0.03">
      <system-out><![CDATA[Hi there!
]]></system-out>
    </testcase>
  </testsuite>
</testsuites>
```
  • Loading branch information
fzakaria committed Oct 11, 2024
1 parent 43d61cb commit 28684ad
Show file tree
Hide file tree
Showing 7 changed files with 232 additions and 5 deletions.
1 change: 1 addition & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ dev_maven.install(
"org.junit.platform:junit-platform-suite:1.8.2",
"org.junit.platform:junit-platform-suite-api:1.8.2",
"org.junit.platform:junit-platform-suite-engine:1.8.2",
"org.junit.platform:junit-platform-testkit:1.8.2",
"org.junit.vintage:junit-vintage-engine:5.8.2",
"org.mockito:mockito-core:4.8.1",
],
Expand Down
1 change: 1 addition & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ maven_install(
"org.junit.platform:junit-platform-suite:1.8.2",
"org.junit.platform:junit-platform-suite-api:1.8.2",
"org.junit.platform:junit-platform-suite-engine:1.8.2",
"org.junit.platform:junit-platform-testkit:1.8.2",
"org.junit.vintage:junit-vintage-engine:5.8.2",
"org.mockito:mockito-core:4.8.1",
],
Expand Down
136 changes: 134 additions & 2 deletions contrib_rules_jvm_tests_install.json
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL",
"__INPUT_ARTIFACTS_HASH": -579211453,
"__RESOLVED_ARTIFACTS_HASH": -1476897549,
"__INPUT_ARTIFACTS_HASH": -110331756,
"__RESOLVED_ARTIFACTS_HASH": 1220653889,
"artifacts": {
"junit:junit": {
"shasums": {
Expand Down Expand Up @@ -31,6 +31,13 @@
},
"version": "1.1.2"
},
"org.assertj:assertj-core": {
"shasums": {
"jar": "d749db58c2bd353f1c03541d747b753931d4b84da8e48993ef51efe8694b4ed7",
"sources": "d0384d378df4391392bbdf06691aa48b3ac3bc5f37c223e6466a75a03d54fee4"
},
"version": "3.20.2"
},
"org.hamcrest:hamcrest-core": {
"shasums": {
"jar": "66fdef91e9739348df7a096aa384a5685f4e875584cce89386a7a47251c4d8e9",
Expand Down Expand Up @@ -115,6 +122,13 @@
},
"version": "1.8.2"
},
"org.junit.platform:junit-platform-testkit": {
"shasums": {
"jar": "94d6fb9b1d4e7bb3d6a967b21f1cf4f7cf84455d602cccf614a186d936929b08",
"sources": "485fac0cc55c14db0c10dff3e42351b10e0bfe5d337892efe8e5b5fcb672f752"
},
"version": "1.8.2"
},
"org.junit.vintage:junit-vintage-engine": {
"shasums": {
"jar": "ebd567b84e380d5373c47de3c9616d84f7bef91f9f8a8e7fc925be68240c1ba4",
Expand Down Expand Up @@ -198,6 +212,12 @@
"org.junit.platform:junit-platform-suite-api",
"org.junit.platform:junit-platform-suite-commons"
],
"org.junit.platform:junit-platform-testkit": [
"org.apiguardian:apiguardian-api",
"org.assertj:assertj-core",
"org.junit.platform:junit-platform-launcher",
"org.opentest4j:opentest4j"
],
"org.junit.vintage:junit-vintage-engine": [
"junit:junit",
"org.apiguardian:apiguardian-api",
Expand Down Expand Up @@ -294,6 +314,69 @@
"org.apiguardian:apiguardian-api": [
"org.apiguardian.api"
],
"org.assertj:assertj-core": [
"org.assertj.core.annotations",
"org.assertj.core.api",
"org.assertj.core.api.exception",
"org.assertj.core.api.filter",
"org.assertj.core.api.iterable",
"org.assertj.core.api.junit.jupiter",
"org.assertj.core.api.recursive.comparison",
"org.assertj.core.condition",
"org.assertj.core.configuration",
"org.assertj.core.data",
"org.assertj.core.description",
"org.assertj.core.error",
"org.assertj.core.error.array2d",
"org.assertj.core.error.future",
"org.assertj.core.error.uri",
"org.assertj.core.extractor",
"org.assertj.core.groups",
"org.assertj.core.internal",
"org.assertj.core.internal.bytebuddy",
"org.assertj.core.internal.bytebuddy.agent.builder",
"org.assertj.core.internal.bytebuddy.asm",
"org.assertj.core.internal.bytebuddy.build",
"org.assertj.core.internal.bytebuddy.description",
"org.assertj.core.internal.bytebuddy.description.annotation",
"org.assertj.core.internal.bytebuddy.description.enumeration",
"org.assertj.core.internal.bytebuddy.description.field",
"org.assertj.core.internal.bytebuddy.description.method",
"org.assertj.core.internal.bytebuddy.description.modifier",
"org.assertj.core.internal.bytebuddy.description.type",
"org.assertj.core.internal.bytebuddy.dynamic",
"org.assertj.core.internal.bytebuddy.dynamic.loading",
"org.assertj.core.internal.bytebuddy.dynamic.scaffold",
"org.assertj.core.internal.bytebuddy.dynamic.scaffold.inline",
"org.assertj.core.internal.bytebuddy.dynamic.scaffold.subclass",
"org.assertj.core.internal.bytebuddy.implementation",
"org.assertj.core.internal.bytebuddy.implementation.attribute",
"org.assertj.core.internal.bytebuddy.implementation.auxiliary",
"org.assertj.core.internal.bytebuddy.implementation.bind",
"org.assertj.core.internal.bytebuddy.implementation.bind.annotation",
"org.assertj.core.internal.bytebuddy.implementation.bytecode",
"org.assertj.core.internal.bytebuddy.implementation.bytecode.assign",
"org.assertj.core.internal.bytebuddy.implementation.bytecode.assign.primitive",
"org.assertj.core.internal.bytebuddy.implementation.bytecode.assign.reference",
"org.assertj.core.internal.bytebuddy.implementation.bytecode.collection",
"org.assertj.core.internal.bytebuddy.implementation.bytecode.constant",
"org.assertj.core.internal.bytebuddy.implementation.bytecode.member",
"org.assertj.core.internal.bytebuddy.jar.asm",
"org.assertj.core.internal.bytebuddy.jar.asm.commons",
"org.assertj.core.internal.bytebuddy.jar.asm.signature",
"org.assertj.core.internal.bytebuddy.matcher",
"org.assertj.core.internal.bytebuddy.pool",
"org.assertj.core.internal.bytebuddy.utility",
"org.assertj.core.internal.bytebuddy.utility.privilege",
"org.assertj.core.internal.bytebuddy.utility.visitor",
"org.assertj.core.matcher",
"org.assertj.core.presentation",
"org.assertj.core.util",
"org.assertj.core.util.diff",
"org.assertj.core.util.diff.myers",
"org.assertj.core.util.introspection",
"org.assertj.core.util.xml"
],
"org.hamcrest:hamcrest-core": [
"org.hamcrest",
"org.hamcrest.core",
Expand Down Expand Up @@ -380,6 +463,9 @@
"org.junit.platform:junit-platform-suite-engine": [
"org.junit.platform.suite.engine"
],
"org.junit.platform:junit-platform-testkit": [
"org.junit.platform.testkit.engine"
],
"org.junit.vintage:junit-vintage-engine": [
"org.junit.vintage.engine",
"org.junit.vintage.engine.descriptor",
Expand Down Expand Up @@ -479,6 +565,8 @@
"net.bytebuddy:byte-buddy:jar:sources",
"org.apiguardian:apiguardian-api",
"org.apiguardian:apiguardian-api:jar:sources",
"org.assertj:assertj-core",
"org.assertj:assertj-core:jar:sources",
"org.hamcrest:hamcrest-core",
"org.hamcrest:hamcrest-core:jar:sources",
"org.junit.jupiter:junit-jupiter-api",
Expand All @@ -503,6 +591,8 @@
"org.junit.platform:junit-platform-suite-engine",
"org.junit.platform:junit-platform-suite-engine:jar:sources",
"org.junit.platform:junit-platform-suite:jar:sources",
"org.junit.platform:junit-platform-testkit",
"org.junit.platform:junit-platform-testkit:jar:sources",
"org.junit.vintage:junit-vintage-engine",
"org.junit.vintage:junit-vintage-engine:jar:sources",
"org.mockito:mockito-core",
Expand All @@ -513,5 +603,47 @@
"org.opentest4j:opentest4j:jar:sources"
]
},
"services": {
"org.junit.jupiter:junit-jupiter-engine": {
"org.junit.platform.engine.TestEngine": [
"org.junit.jupiter.engine.JupiterTestEngine"
]
},
"org.junit.jupiter:junit-jupiter-engine:jar:sources": {
"org.junit.platform.engine.TestEngine": [
"org.junit.jupiter.engine.JupiterTestEngine"
]
},
"org.junit.platform:junit-platform-launcher": {
"org.junit.platform.launcher.TestExecutionListener": [
"org.junit.platform.launcher.listeners.UniqueIdTrackingListener"
]
},
"org.junit.platform:junit-platform-launcher:jar:sources": {
"org.junit.platform.launcher.TestExecutionListener": [
"org.junit.platform.launcher.listeners.UniqueIdTrackingListener"
]
},
"org.junit.platform:junit-platform-suite-engine": {
"org.junit.platform.engine.TestEngine": [
"org.junit.platform.suite.engine.SuiteTestEngine"
]
},
"org.junit.platform:junit-platform-suite-engine:jar:sources": {
"org.junit.platform.engine.TestEngine": [
"org.junit.platform.suite.engine.SuiteTestEngine"
]
},
"org.junit.vintage:junit-vintage-engine": {
"org.junit.platform.engine.TestEngine": [
"org.junit.vintage.engine.VintageTestEngine"
]
},
"org.junit.vintage:junit-vintage-engine:jar:sources": {
"org.junit.platform.engine.TestEngine": [
"org.junit.vintage.engine.VintageTestEngine"
]
}
},
"version": "2"
}
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ private Map<TestData, List<TestData>> matchTestCasesToSuites_locked(
// results
List<UniqueId.Segment> segments = testCase.getId().getUniqueIdObject().getSegments();

if (segments.size() == 3 || segments.size() == 5) {
if (segments.size() == 2) {
parent = results.get(testCase.getId().getUniqueIdObject());
} else if (segments.size() == 3 || segments.size() == 5) {
// get class / test data up one level
parent =
testCase
Expand Down Expand Up @@ -161,7 +163,7 @@ private List<TestData> findTestCases_locked() {
// are identified by the fact that they have no child test cases in the
// test plan, or they are marked as tests.
TestIdentifier id = result.getId();
return id.isTest() || testPlan.getChildren(id).isEmpty();
return id.getSource() != null || id.isTest() || testPlan.getChildren(id).isEmpty();
})
.collect(Collectors.toList());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,4 +146,25 @@ public boolean isDynamic() {
public Instant getStarted() {
return this.started;
}

@Override
public String toString() {
return "TestData{"
+ "id="
+ id
+ ", reportEntries="
+ reportEntries
+ ", started="
+ started
+ ", finished="
+ finished
+ ", reason='"
+ reason
+ '\''
+ ", result="
+ result
+ ", dynamic="
+ dynamic
+ '}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ java_test_suite(
artifact("org.junit.jupiter:junit-jupiter-api", "contrib_rules_jvm_tests"),
artifact("org.junit.jupiter:junit-jupiter-params", "contrib_rules_jvm_tests"),
artifact("org.junit.platform:junit-platform-engine", "contrib_rules_jvm_tests"),
artifact("org.junit.platform:junit-platform-testkit", "contrib_rules_jvm_tests"),
artifact("org.mockito:mockito-core", "contrib_rules_jvm_tests"),
artifact("org.opentest4j:opentest4j", "contrib_rules_jvm_tests"),
] + junit5_vintage_deps("contrib_rules_jvm_tests"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,39 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
import static org.junit.platform.launcher.LauncherConstants.STDERR_REPORT_ENTRY_KEY;
import static org.junit.platform.launcher.LauncherConstants.STDOUT_REPORT_ENTRY_KEY;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.nio.file.Files;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.junit.platform.engine.TestDescriptor;
import org.junit.platform.engine.TestExecutionResult;
import org.junit.platform.engine.UniqueId;
import org.junit.platform.engine.reporting.ReportEntry;
import org.junit.platform.launcher.Launcher;
import org.junit.platform.launcher.TestIdentifier;
import org.junit.platform.launcher.TestPlan;
import org.junit.platform.launcher.core.LauncherConfig;
import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
import org.junit.platform.launcher.core.LauncherFactory;
import org.junit.platform.testkit.engine.EngineTestKit;
import org.mockito.Mockito;
import org.opentest4j.TestAbortedException;
import org.w3c.dom.Document;
Expand All @@ -37,11 +47,70 @@
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

public class BazelJUnitOuputListenerTest {
public class BazelJUnitOutputListenerTest {

private TestDescriptor testDescriptor = new StubbedTestDescriptor(createId("descriptors"));
private TestIdentifier identifier = TestIdentifier.from(testDescriptor);

/** This latch is used in TestAfterAllFails for testAfterAllFailuresAreReported */
private static final AtomicBoolean causeFailure = new AtomicBoolean(false);

static final class TestAfterAllFails {
@AfterAll
static void afterAll() {
if (causeFailure.get()) {
throw new RuntimeException("I always fail.");
}
}

@Test
void test() {}
}

@Test
public void testAfterAllFailuresAreReported() throws Exception {
causeFailure.set(true);

// First let's do a sanity test that we have the expected failures for the @AfterAll
EngineTestKit.engine("junit-jupiter")
.selectors(selectClass(TestAfterAllFails.class))
.execute()
.containerEvents()
.assertStatistics(stats -> stats.skipped(0).started(2).succeeded(1).aborted(0).failed(1));

// Now let's run the same test. Unfortunately we cannot use EngineTestKit since it has no way
// to register a listener.
File xmlFile = File.createTempFile("junit-report", "xml");
BazelJUnitOutputListener listener = new BazelJUnitOutputListener(xmlFile.toPath());
LauncherConfig config = LauncherConfig.builder().addTestExecutionListeners(listener).build();

Launcher launcher = LauncherFactory.create(config);

LauncherDiscoveryRequestBuilder request =
LauncherDiscoveryRequestBuilder.request().selectors(selectClass(TestAfterAllFails.class));

TestPlan plan = launcher.discover(request.build());

launcher.execute(request.build());
listener.close();

// now write an assertion to validate the XML file has an error
String[] expectedStrings = {
"<failure message=\"I always fail.\" type=\"java.lang.RuntimeException\">", "failures=\"1\"",
};

// Useful for debugging the expected output
// System.out.println(Files.readString(xmlFile.toPath()));

for (String expected : expectedStrings) {
assertTrue(
Files.lines(xmlFile.toPath()).anyMatch(line -> line.contains(expected)),
"Expected to find " + expected + " in " + xmlFile);
}

causeFailure.set(false);
}

@Test
public void testResultCanBeDisabled() {
// Note: we do not call `markFinished` so the TestResult is null. This is what happens when
Expand Down

0 comments on commit 28684ad

Please sign in to comment.