Allow running tests more silently

When running tests at present via `make test`, the developer is met with
lots of messages from both the testing script and logger messages
emitted through libraries called by the testcases themselves. This
clutters the output and makes it hard to distinguish actual test suite
failures from regular progress and other logging messages.

This commit introduces support for interpreting the `-s` / `--silent` /
`--quiet` option from `make` and utilizing it to run tests in a more
quiet manner. Specifically, this means that log output is fully captured
by `cth_log_redirect` (instead of logging to both stdout and the CT
log), and the banners and disclaimers from the testing script are
reduced to a minimum (under the assumption that the person running the
tests is familar with them).

To accomplish this, the `cth_log_redirect` handler is augmented with the
`mode` option, which instructs it how to work. The previous behaviour -
logging to both stdout and the Common Test logs - is kept by default for
now.

As a small improvement that came up whilst implementing this change,
`logger_std_h` is updated to skip logging filesync errors when
instructeed to log to `/dev/null`.

In `inets`, this reduces the output emitted by the tests from 2175 to
803 lines.

Author: jchristgit

HOWTO/DEVELOPMENT.md

@@ -213,6 +213,14 @@ for all process you would do this:
 ERL_ARGS="+hmqd off_heap" make emulator_test
 ```
 
+If you want logger messages to only be printed into the Common Test HTML logs
+and keep test script information about starting and stopping the run to a
+minimum, you can pass `-s` or `--silent`:
+
+```bash
+make test -s
+```
+
 ### Build and test a specific application
 
 You can also build the application from within itself. Like this:

Makefile.in

@@ -1007,8 +1007,12 @@ bootstrap_clean:
 
 .PHONY: test dialyzer
 
+ifeq ($(findstring s,${MAKEFLAGS}),s)
+	TEST_EXTRA_ARGS = --quiet
+endif
+
 test: all release release_tests
-	$(ERL_TOP)/make/test_target_script.sh $(ERL_TOP)
+	$(ERL_TOP)/make/test_target_script.sh $(TEST_EXTRA_ARGS) $(ERL_TOP)
 
 ifeq ($(TYPE),)
 dialyzer: all

lib/common_test/doc/src/ct_hooks_chapter.xml

@@ -492,6 +492,22 @@ results(State) ->
         use another level either change the <c>default</c> handler level before
         starting common_test, or use the <seemfa marker="kernel:logger#set_handler_config/3">
         <c>logger:set_handler_config/3</c></seemfa> API.</p>
+        <p>This hook supports the following options:</p>
+        <taglist>
+          <tag><c>{mode, replace}</c></tag>
+          <item>
+            <p>Replace the <c>default</c> logging handler by the log redirect
+              instead of logging to both the default handler and the
+              <c>cth_log_redirect</c> handler. To use this mode, disable the builtin
+              hook and reconfigure it:</p>
+            <p><c>-enable_builtin_hooks false -ct_hooks cth_log_redirect [{mode,replace}]</c></p>
+          </item>
+          <tag><c>{mode, add}</c></tag>
+          <item>
+            <p>Add the logging handler instead of replacing the default logging
+              handler. This is the default behaviour.</p>
+          </item>
+        </taglist>
        </item>
        <tag><c>cth_surefire</c></tag>
        <item>

lib/common_test/src/ct_hooks.erl

@@ -36,7 +36,7 @@
 %% All of the hooks which are to be started by default. Remove by issuing
 %% -enable_builtin_hooks false to when starting common test.
 -define(BUILTIN_HOOKS,[#ct_hook_config{ module = cth_log_redirect,
-					opts = [],
+					opts = [{mode, add}],
 					prio = ctfirst }]).
 
 -record(ct_hook_config, {id, module, prio, scope, opts = [],

lib/common_test/src/cth_log_redirect.erl

@@ -21,7 +21,8 @@
 
 %%% Common Test Framework functions handling test specifications.
 %%%
-%%% This module redirects sasl and error logger info to common test log.
+%%% This module redirects sasl, error logger, and standard logger messages to
+%%% the common test log.
 
 %% CTH Callbacks
 -export([id/1, init/2,
@@ -55,10 +56,10 @@
 id(_Opts) ->
     ?MODULE.
 
-init(?MODULE, _Opts) ->
+init(?MODULE, Opts) ->
     ct_util:mark_process(),
-    ok = start_log_handler(),
-    tc_log_async.
+    ok = start_log_handler(Opts),
+    {ok, tc_log_async}.
 
 pre_init_per_suite(Suite, Config, State) ->
     set_curr_func({Suite,init_per_suite}, Config),
@@ -114,7 +115,7 @@ post_end_per_group(_Suite, _Group, Config, Return, State) ->
     set_curr_func({group,undefined}, Config),
     {Return, State}.
 
-start_log_handler() ->
+start_log_handler(Options) ->
     case whereis(?MODULE) of
         undefined ->
             ChildSpec =
@@ -136,9 +137,14 @@ start_log_handler() ->
             _Else ->
                 {{?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG},info}
         end,
-    ok = logger:add_handler(?MODULE,?MODULE,
-                            #{level=>DefaultLevel,
-                              formatter=>DefaultFormatter}).
+    HandlerConfig = #{level => DefaultLevel, formatter => DefaultFormatter},
+    case proplists:get_value(mode, Options) of
+        add ->
+            ok = logger:add_handler(?MODULE, ?MODULE, HandlerConfig);
+        replace ->
+            ok = logger:remove_handler(default),
+            ok = logger:add_handler(default, ?MODULE, HandlerConfig)
+    end.
 
 init([]) ->
     {ok, #eh_state{log_func = tc_log_async}}.
@@ -171,8 +177,6 @@ log(#{msg:={report,Msg},meta:=#{domain:=[otp,sasl]}}=Log,Config) ->
     end;
 log(#{meta:=#{domain:=[otp]}}=Log,Config) ->
     do_log(add_log_category(Log,error_logger),Config);
-log(#{meta:=#{domain:=_}},_) ->
-    ok;
 log(Log,Config) ->
     do_log(add_log_category(Log,error_logger),Config).
 

lib/kernel/src/logger_std_h.erl

@@ -673,6 +673,10 @@ maybe_notify_error(Op, Result, #{write_res:=WR,sync_res:=SR})
        (Op==filesync andalso Result==SR) ->
     %% don't report same error twice
     ok;
+maybe_notify_error(filesync, {error, einval}, #{file_name:="/dev/null"}) ->
+    %% If logging to /dev/null, output should be discarded
+    %% Ignore any EINVAL errors for filesync operations
+    ok;
 maybe_notify_error(Op, Error, #{handler_name:=HandlerName,file_name:=FileName}) ->
     logger_h_common:error_notify({HandlerName,Op,FileName,Error}),
     ok.

make/app_targets.mk

@@ -23,8 +23,13 @@ APPLICATION ?= $(basename $(notdir $(PWD)))
 .PHONY: test info gclean dialyzer dialyzer_plt dclean
 
 ifndef NO_TEST_TARGET
+
+ifeq ($(findstring s,${MAKEFLAGS}),s)
+TEST_ENV = QUIET=1
+endif
+
 test:
-	TEST_NEEDS_RELEASE=$(TEST_NEEDS_RELEASE) TYPE=$(TYPE) \
+	TEST_NEEDS_RELEASE=$(TEST_NEEDS_RELEASE) TYPE=$(TYPE) $(TEST_ENV) \
 	  $(ERL_TOP)/make/test_target_script.sh $(ERL_TOP)
 endif
 

make/test_target_script.sh

@@ -27,15 +27,27 @@ LIGHT_CYAN='\033[1;36m'
 BOLD='\033[1m'
 NC='\033[0m'
 
+QUIET="${QUIET:-0}"
+if [ "$QUIET" -eq 1 ] && [ "${WSLcross}" != "true" ]; then
+    # cth_log_redirect is enabled by default, to configure it we need to remove and re-add it
+    ARGS="-enable_builtin_hooks false -ct_hooks cth_log_redirect [{mode,replace}] ${ARGS}"
+fi
+
 
 print_highlighted_msg_with_printer () {
     COLOR=$1
     MSG_PRINTER=$2
-    printf "\n${COLOR}======================================================================${NC}\n"
-    echo
-    $MSG_PRINTER
-    echo
-    printf "${COLOR}======================================================================${NC}\n"
+    IMPORTANT=${3:-0}
+    if [ "$QUIET" -ne 1 ]; then
+        printf "\n${COLOR}======================================================================${NC}\n"
+        echo
+    fi
+    if [ "$QUIET" -ne 1 ] || [ "$IMPORTANT" -eq 1 ]; then
+        $MSG_PRINTER
+    fi
+    if [ "$QUIET" -ne 1 ]; then
+        printf "${COLOR}======================================================================${NC}\n"
+    fi
 }
 
 print_highlighted_msg () {
@@ -44,7 +56,7 @@ print_highlighted_msg () {
     print_msg () {
         echo "$MSG"
     }
-    print_highlighted_msg_with_printer $COLOR print_msg
+    print_highlighted_msg_with_printer $COLOR print_msg 1
 }
 
 print_all_tests_takes_long_time_warning () {