ztest: Fix confusing SKIP log

When CONFIG_ZTEST_FAIL_ON_ASSUME is set, a failed assumption will
cause the suite to fail, but the individual test will be marked as
SKIPPED. We should fail the test so it's clear what's going on.

Fixes zephyrproject-rtos#86611

Signed-off-by: Yuval Peress <[email protected]>

Author: yperess

subsys/testsuite/ztest/Kconfig

@@ -208,7 +208,6 @@ config ZTEST_VERBOSE_SUMMARY
 
 config ZTEST_FAIL_ON_ASSUME
 	bool "Fail the test run when an assumption fails"
-	default y
 	help
 	  When enabled, the test binary will fail at the end if an assumption failed. This means
 	  that while tests will still be marked as skipped on failed zassume calls, the final test

subsys/testsuite/ztest/src/ztest.c

@@ -393,8 +393,10 @@ void ztest_skip_failed_assumption(void)
 {
 	if (IS_ENABLED(CONFIG_ZTEST_FAIL_ON_ASSUME)) {
 		current_test_failed_assumption = true;
+		ztest_test_fail();
+	} else {
+		ztest_test_skip();
 	}
-	ztest_test_skip();
 }
 
 #ifndef KERNEL

tests/ztest/fail/CMakeLists.txt

@@ -30,7 +30,9 @@ string(REGEX MATCHALL "(^|;)CONFIG_ZTEST_FAIL_TEST_[A-Za-z0-9_]+" fail_test_conf
 list(FILTER fail_test_config EXCLUDE REGEX "^$")
 list(TRANSFORM fail_test_config PREPEND "-D")
 list(TRANSFORM fail_test_config APPEND "=y")
-string(REPLACE ";" " " fail_test_config "${fail_test_config}")
+if(CONFIG_ZTEST_FAIL_ON_ASSUME)
+  list(APPEND fail_test_config "-DCONFIG_ZTEST_FAIL_ON_ASSUME=y")
+endif()
 
 # Add the 'core' external project which will mirror the configs of this project.
 ExternalProject_Add(core

tests/ztest/fail/Kconfig

@@ -31,8 +31,10 @@ config TEST_ERROR_STRING
 	string
 	default "ERROR: cannot fail in test phase 'after()', bailing" if ZTEST_FAIL_TEST_ASSERT_AFTER
 	default "ERROR: cannot fail in test phase 'teardown()', bailing" if ZTEST_FAIL_TEST_ASSERT_TEARDOWN
-	default "ERROR: cannot skip in test phase 'after()', bailing" if ZTEST_FAIL_TEST_ASSUME_AFTER
-	default "ERROR: cannot skip in test phase 'teardown()', bailing" if ZTEST_FAIL_TEST_ASSUME_TEARDOWN
+	default "ERROR: cannot skip in test phase 'after()', bailing" if ZTEST_FAIL_TEST_ASSUME_AFTER && !ZTEST_FAIL_ON_ASSUME
+	default "ERROR: cannot skip in test phase 'teardown()', bailing" if ZTEST_FAIL_TEST_ASSUME_TEARDOWN && !ZTEST_FAIL_ON_ASSUME
+	default "ERROR: cannot fail in test phase 'after()', bailing" if ZTEST_FAIL_TEST_ASSUME_AFTER && ZTEST_FAIL_ON_ASSUME
+	default "ERROR: cannot fail in test phase 'teardown()', bailing" if ZTEST_FAIL_TEST_ASSUME_TEARDOWN && ZTEST_FAIL_ON_ASSUME
 	default "ERROR: cannot pass in test phase 'after()', bailing" if ZTEST_FAIL_TEST_PASS_AFTER
 	default "ERROR: cannot pass in test phase 'teardown()', bailing" if ZTEST_FAIL_TEST_PASS_TEARDOWN
 	default "Assumption failed at " if ZTEST_FAIL_TEST_UNEXPECTED_ASSUME

tests/ztest/fail/core/CMakeLists.txt

@@ -23,6 +23,10 @@ elseif(CONFIG_ZTEST_FAIL_TEST_UNEXPECTED_ASSUME)
   list(APPEND test_sources src/unexpected_assume.cpp)
 endif()
 
+if(CONFIG_ZTEST_FAIL_ON_ASSUME)
+  add_definitions(-DCONFIG_ZTEST_FAIL_ON_ASSUME=1)
+endif()
+
 if(BOARD STREQUAL "unit_testing" OR BOARD STREQUAL "unit_testing/unit_testing")
   find_package(Zephyr COMPONENTS unittest REQUIRED HINTS $ENV{ZEPHYR_BASE})
   project(base)

tests/ztest/fail/testcase.yaml

@@ -42,12 +42,22 @@ tests:
   testing.fail.zephyr.assert_teardown:
     extra_configs:
       - CONFIG_ZTEST_FAIL_TEST_ASSERT_TEARDOWN=y
-  testing.fail.zephyr.assume_after:
+  testing.fail.zephyr.assume_after.fail:
     extra_configs:
       - CONFIG_ZTEST_FAIL_TEST_ASSUME_AFTER=y
-  testing.fail.zephyr.assume_teardown:
+      - CONFIG_ZTEST_FAIL_ON_ASSUME=y
+  testing.fail.zephyr.assume_teardown.fail:
     extra_configs:
       - CONFIG_ZTEST_FAIL_TEST_ASSUME_TEARDOWN=y
+      - CONFIG_ZTEST_FAIL_ON_ASSUME=y
+  testing.fail.zephyr.assume_after.skip:
+    extra_configs:
+      - CONFIG_ZTEST_FAIL_TEST_ASSUME_AFTER=y
+      - CONFIG_ZTEST_FAIL_ON_ASSUME=n
+  testing.fail.zephyr.assume_teardown.skip:
+    extra_configs:
+      - CONFIG_ZTEST_FAIL_TEST_ASSUME_TEARDOWN=y
+      - CONFIG_ZTEST_FAIL_ON_ASSUME=n
   testing.fail.zephyr.pass_after:
     extra_configs:
       - CONFIG_ZTEST_FAIL_TEST_PASS_AFTER=y
@@ -58,6 +68,8 @@ tests:
     type: unit
     extra_configs:
       - CONFIG_ZTEST_FAIL_TEST_UNEXPECTED_ASSUME=y
+      - CONFIG_ZTEST_FAIL_ON_ASSUME=y
   testing.fail.zephyr.fail_on_bad_assumption:
     extra_configs:
       - CONFIG_ZTEST_FAIL_TEST_UNEXPECTED_ASSUME=y
+      - CONFIG_ZTEST_FAIL_ON_ASSUME=y