Fix: Final test result for JUnit tests failed with exception (#3299)

serenity-model/src/main/java/net/thucydides/model/domain/TestResult.java

@@ -15,62 +15,56 @@
  * and of the overall acceptance test case itself.
  * 
  * @author johnsmart
- *
  */
 public enum TestResult {
     /**
-     * Either failure, error or compromised - internal use only
-     */
-    UNSUCCESSFUL(6, true, "Unsuccessful"),
-    /**
-     * Test failures due to external events or systems that compromise the validity of the test.
+     * Test result not known yet.
      */
-    COMPROMISED(5, true, "Compromised"),
+    UNDEFINED(0, false, "Undefined"),
     /**
-     *  Test failure, due to some other exception.
+     * The test or test case ran as expected.
      */
-     ERROR(4, true, "Broken"),
+    SUCCESS(1, true, "Passing"),
     /**
-     * Test failure, due to an assertion error
-     * For a test case, this means one of the tests in the test case failed.
+     * A pending test is one that has been specified but not yet implemented.
+     * In a JUnit test case, you can use the (Thucydides) @Pending annotation to mark this.
+     * A pending test case is a test case that has at least one pending test.
      */
-    FAILURE(3, true, "Failing"),
-
+    PENDING(2, false, "Pending"),
     /**
-     * Test is skipped due to a failing assumption
+     * The test or test case was deliberately ignored.
+     * Tests can be ignored via the @Ignore annotation in JUnit, for example.
+     * Ignored tests are not considered the same as pending tests: a pending test is one that
+     * has been specified, but the corresponding code is yet to be implemented, whereas an
+     * ignored test can be a temporarily-deactivated test (during refactoring, for example).
      */
-    ABORTED(2, false, "Aborted"),
-
+    IGNORED(2, false, "Ignored"),
     /**
      * The test step was not executed because a previous step in this test case failed.
      * A whole test case can be skipped using tags or annotations to indicate that it is currently "work-in-progress"
      */
     SKIPPED(2, false, "Skipped"),
     /**
-     * The test or test case was deliberately ignored.
-     * Tests can be ignored via the @Ignore annotation in JUnit, for example.
-     * Ignored tests are not considered the same as pending tests: a pending test is one that
-     * has been specified, but the corresponding code is yet to be implemented, whereas an
-     * ignored test can be a temporarily-deactivated test (during refactoring, for example).
+     * Test is skipped due to a failing assumption
      */
-    IGNORED(2, false, "Ignored"),
-
+    ABORTED(2, false, "Aborted"),
     /**
-     * A pending test is one that has been specified but not yet implemented.
-     * In a JUnit test case, you can use the (Thucydides) @Pending annotation to mark this.
-     * A pending test case is a test case that has at least one pending test.
+     * Test failure, due to an assertion error
+     * For a test case, this means one of the tests in the test case failed.
      */
-    PENDING(2, false, "Pending"),
-
+    FAILURE(3, true, "Failing"),
     /**
-     * The test or test case ran as expected.
+     * Test failure, due to some other exception.
      */
-    SUCCESS(1, true, "Passing"),
-
+    ERROR(4, true, "Broken"),
     /**
-     * Test result not known yet.
+     * Test failures due to external events or systems that compromise the validity of the test.
+     */
+    COMPROMISED(5, true, "Compromised"),
+    /**
+     * Either failure, error or compromised - internal use only
      */
-    UNDEFINED(0, false, "Undefined");
+    UNSUCCESSFUL(6, true, "Unsuccessful");
 
     private final int priority;
     private final boolean executedResultsCount;

serenity-model/src/test/java/net/thucydides/model/domain/TestResultTest.java

@@ -0,0 +1,68 @@
+package net.thucydides.model.domain;
+
+import org.junit.experimental.runners.Enclosed;
+import org.junit.jupiter.api.Assertions;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+@RunWith(Enclosed.class)
+public class TestResultTest {
+
+    @RunWith(Parameterized.class)
+    public static class IsMoreSevereTest {
+        private final TestResult firstStatus;
+        private final TestResult secondStatus;
+        private final boolean expectedResult;
+
+        public IsMoreSevereTest(TestResult firstStatus, TestResult secondStatus, boolean expectedResult) {
+            this.firstStatus = firstStatus;
+            this.secondStatus = secondStatus;
+            this.expectedResult = expectedResult;
+        }
+
+        @Test
+        public void should_detect_more_severe_results() {
+            Assertions.assertEquals(firstStatus.isMoreSevereThan(secondStatus), expectedResult);
+        }
+
+        @Parameterized.Parameters
+        public static Collection<Object[]> data() {
+            return Arrays.asList(new Object[][]{
+                    {TestResult.FAILURE, TestResult.SUCCESS, true},
+                    {TestResult.FAILURE, TestResult.FAILURE, false},
+                    {TestResult.SUCCESS, TestResult.FAILURE, false}
+            });
+        }
+    }
+
+    @RunWith(Parameterized.class)
+    public static class IsLessSevereTest {
+        private final TestResult firstStatus;
+        private final TestResult secondStatus;
+        private final boolean expectedResult;
+
+        public IsLessSevereTest(TestResult firstStatus, TestResult secondStatus, boolean expectedResult) {
+            this.firstStatus = firstStatus;
+            this.secondStatus = secondStatus;
+            this.expectedResult = expectedResult;
+        }
+
+        @Test
+        public void should_detect_less_severe_results() {
+            Assertions.assertEquals(firstStatus.isLessSevereThan(secondStatus), expectedResult);
+        }
+
+        @Parameterized.Parameters
+        public static Collection<Object[]> data() {
+            return Arrays.asList(new Object[][]{
+                    {TestResult.FAILURE, TestResult.SUCCESS, false},
+                    {TestResult.FAILURE, TestResult.FAILURE, false},
+                    {TestResult.SUCCESS, TestResult.FAILURE, true}
+            });
+        }
+    }
+}