diff --git a/lib/ceedling/generator.rb b/lib/ceedling/generator.rb index 99e478e2..9ead6d38 100644 --- a/lib/ceedling/generator.rb +++ b/lib/ceedling/generator.rb @@ -26,7 +26,7 @@ class Generator :loginator, :plugin_manager, :file_wrapper, - :unity_utils + :test_runner_manager def setup() @@ -291,13 +291,14 @@ def generate_test_results(tool:, context:, test_name:, test_filepath:, executabl msg = @reportinator.generate_progress( "Running #{File.basename(arg_hash[:executable])}" ) @loginator.log( msg ) - # Unity's exit code is equivalent to the number of failed tests, so we tell @tool_executor not to fail out if there are failures - # so that we can run all tests and collect all results + # Unity's exit code is equivalent to the number of failed tests. + # We tell @tool_executor not to fail out if there are failures + # so that we can run all tests and collect all results. command = @tool_executor.build_command_line( arg_hash[:tool], # Apply additional test case filters - @unity_utils.collect_test_runner_additional_args(), + @test_runner_manager.collect_cmdline_args(), arg_hash[:executable] ) @@ -312,7 +313,10 @@ def generate_test_results(tool:, context:, test_name:, test_filepath:, executabl @helper.log_test_results_crash( test_name, executable, shell_result ) filename = File.basename( test_filepath ) + + # Lookup test cases and filter based on any matchers specified for the build task test_cases = @test_context_extractor.lookup_test_cases( test_filepath ) + test_cases = @generator_test_results.filter_test_cases( test_cases ) case @configurator.project_config_hash[:project_use_backtrace] # If we have the options and tools to learn more, dig into the details diff --git a/lib/ceedling/generator_test_results.rb b/lib/ceedling/generator_test_results.rb index 28dd8c8f..8b3398f3 100644 --- a/lib/ceedling/generator_test_results.rb +++ b/lib/ceedling/generator_test_results.rb @@ -10,6 +10,81 @@ require 'ceedling/constants' require 'ceedling/exceptions' + +## +## Sample Unity Test Executable Output +## =================================== +## +## - Output is line-oriented. Anything outside the recognized lines is assumed to be from `printf()` +## or equivalent calls and collected for presentation as a collection of $stdout lines. +## - Multiline output (i.e. failure messages) can be achieved by "encoding" newlines as literal +## "\n"s (slash-n). `extract_line_elements()` handles converting newline markers into real newlines. +## - :PASS has no trailing message unless Unity's test case execution duration feature is enabled. +## If enabled, a numeric value with 'ms' as a units signifier trails, ":PASS 1.2 ms". +## - :IGNORE optionally can include a trailing message. +## - :FAIL has a trailing message that relays an assertion failure or crash condition. +## - The statistics line always has the same format with only the count values varying. +## - If there are no failed test cases, the final line is 'OK'. Otherwise, it is 'FAIL'. +## +## $stdout: +## ----------------------------------------------------------------------------------------------------- +## TestUsartModel.c:24:testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting:PASS +## TestUsartModel.c:34:testIgnore:IGNORE +## TestUsartModel.c:39:testFail:FAIL: Expected 2 Was 3 +## TestUsartModel.c:49:testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately:PASS +## TestUsartModel.c:55:testShouldReturnErrorMessageUponInvalidTemperatureValue:PASS +## TestUsartModel.c:61:testShouldReturnWakeupMessage:PASS +## +## ----------------------- +## 6 Tests 1 Failures 1 Ignored +## FAIL + +## +## Sample Test Results Output File (YAML) +## ====================================== +## The following corresponds to the test executable output above. +## +## TestUsartModel.fail: +## --- +## :source: +## :file: test/TestUsartModel.c +## :dirname: test +## :basename: TestUsartModel.c +## :successes: +## - :test: testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting +## :line: 24 +## :message: '' +## :unity_test_time: 0 +## - :test: testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately +## :line: 49 +## :message: '' +## :unity_test_time: 0 +## - :test: testShouldReturnErrorMessageUponInvalidTemperatureValue +## :line: 55 +## :message: '' +## :unity_test_time: 0 +## - :test: testShouldReturnWakeupMessage +## :line: 61 +## :message: '' +## :unity_test_time: 0 +## :failures: +## - :test: testFail +## :line: 39 +## :message: Expected 2 Was 3 +## :unity_test_time: 0 +## :ignores: +## - :test: testIgnore +## :line: 34 +## :message: '' +## :unity_test_time: 0 +## :counts: +## :total: 6 +## :passed: 4 +## :failed: 1 +## :ignored: 1 +## :stdout: [] +## :time: 0.006512000225484371 + class GeneratorTestResults constructor :configurator, :generator_test_results_sanity_checker, :yaml_wrapper @@ -40,41 +115,68 @@ def process_and_write_results(executable, unity_shell_result, results_file, test end # Remove test statistics lines - output_string = unity_shell_result[:output].sub(TEST_STDOUT_STATISTICS_PATTERN, '') + output_string = unity_shell_result[:output].sub( TEST_STDOUT_STATISTICS_PATTERN, '' ) + + # Process test executable results line-by-line output_string.lines do |line| - # Process Unity output + # Process Unity test executable output case line.chomp when /(:IGNORE)/ elements = extract_line_elements( executable, line, results[:source][:file] ) results[:ignores] << elements[0] results[:stdout] << elements[1] if (!elements[1].nil?) + when /(:PASS$)/ elements = extract_line_elements( executable, line, results[:source][:file] ) results[:successes] << elements[0] results[:stdout] << elements[1] if (!elements[1].nil?) + when /(:PASS \(.* ms\)$)/ elements = extract_line_elements( executable, line, results[:source][:file] ) results[:successes] << elements[0] results[:stdout] << elements[1] if (!elements[1].nil?) + when /(:FAIL)/ elements = extract_line_elements( executable, line, results[:source][:file] ) results[:failures] << elements[0] results[:stdout] << elements[1] if (!elements[1].nil?) - else # Collect up all other output - results[:stdout] << line.chomp + + # Collect up all other output + else + results[:stdout] << line.chomp # Ignores blank lines end end @sanity_checker.verify( results, unity_shell_result[:exit_code] ) - output_file = results_file.ext(@configurator.extension_testfail) if (results[:counts][:failed] > 0) + output_file = results_file.ext( @configurator.extension_testfail ) if (results[:counts][:failed] > 0) @yaml_wrapper.dump(output_file, results) return { :result_file => output_file, :result => results } end - # TODO: Filter test cases with command line test case matchers + # Filter list of test cases: + # --test_case + # --exclude_test_case + # + # @return Array - list of the test_case hashses {:test, :line_number} + def filter_test_cases(test_cases) + _test_cases = test_cases.clone + + # Filter tests which contain test_case_name passed by `--test_case` argument + if !@configurator.include_test_case.empty? + _test_cases.delete_if { |i| !(i[:test] =~ /#{@configurator.include_test_case}/) } + end + + # Filter tests which contain test_case_name passed by `--exclude_test_case` argument + if !@configurator.exclude_test_case.empty? + _test_cases.delete_if { |i| i[:test] =~ /#{@configurator.exclude_test_case}/ } + end + + return _test_cases + end + def create_crash_failure(source, shell_result, test_cases) count = test_cases.size() @@ -96,6 +198,7 @@ def create_crash_failure(source, shell_result, test_cases) return shell_result end + # Fill out a template to mimic Unity's test executable output def regenerate_test_executable_stdout(total:, failed:, ignored:, output:[]) values = { :total => total, diff --git a/lib/ceedling/generator_test_results_backtrace.rb b/lib/ceedling/generator_test_results_backtrace.rb index 72db5dfd..f8ea2cf2 100644 --- a/lib/ceedling/generator_test_results_backtrace.rb +++ b/lib/ceedling/generator_test_results_backtrace.rb @@ -20,8 +20,7 @@ def do_simple(filename, executable, shell_result, test_cases) # Reset time shell_result[:time] = 0 - # Revise test case list with any matches and excludes and iterate - test_cases = filter_test_cases( test_cases ) + # Iterate on test cases test_cases.each do |test_case| # Build the test fixture to run with our test case of interest command = @tool_executor.build_command_line( @@ -83,9 +82,7 @@ def do_gdb(filename, executable, shell_result, test_cases) # Reset time shell_result[:time] = 0 - # Revise test case list with any matches and excludes and iterate - test_cases = filter_test_cases( test_cases ) - + # Iterate on test cases test_cases.each do |test_case| # Build the test fixture to run with our test case of interest command = @tool_executor.build_command_line( @@ -126,20 +123,19 @@ def do_gdb(filename, executable, shell_result, test_cases) # Collect file_name and line in which crash occurred matched = crash_result[:output].match( /#{test_case[:test]}\s*\(\)\sat.+#{filename}:(\d+)\n/ ) - # If we find an error report line containing `test_case() at filename.c:###` + # If we found an error report line containing `test_case() at filename.c:###` in `gdb` output if matched # Line number line_number = matched[1] - # Filter the `gdb` $stdout report + # Filter the `gdb` $stdout report to find most important lines of text crash_report = filter_gdb_test_report( crash_result[:output], test_case[:test], filename ) - # Replace: - # - '\n' by @new_line_tag to make gdb output flat - # - ':' by @colon_tag to avoid test results problems - # to enable parsing output for default generator_test_results regex - # test_output = crash_report.gsub("\n", @new_line_tag).gsub(':', @colon_tag) - test_output = crash_report.gsub( "\n", '\n') + # Unity’s test executable output is line oriented. + # Multi-line output is not possible (it looks like random `printf()` statements to the results parser) + # "Encode" actual newlines as literal "\n"s (slash-n) to be handled by the test results parser. + test_output = crash_report.gsub( "\n", '\n' ) + test_output = "#{filename}:#{line_number}:#{test_case[:test]}:FAIL: Test case crashed >> #{test_output}" # Otherwise communicate that `gdb` failed to produce a usable report @@ -160,34 +156,13 @@ def do_gdb(filename, executable, shell_result, test_cases) failed: test_case_results[:failed], output: test_case_results[:output] ) - puts shell_result[:output] + return shell_result end ### Private ### private - # Filter list of test cases: - # --test_case - # --exclude_test_case - # - # @return Array - list of the test_case hashses {:test, :line_number} - def filter_test_cases(test_cases) - _test_cases = test_cases.clone - - # Filter tests which contain test_case_name passed by `--test_case` argument - if !@configurator.include_test_case.empty? - _test_cases.delete_if { |i| !(i[:test] =~ /#{@configurator.include_test_case}/) } - end - - # Filter tests which contain test_case_name passed by `--exclude_test_case` argument - if !@configurator.exclude_test_case.empty? - _test_cases.delete_if { |i| i[:test] =~ /#{@configurator.exclude_test_case}/ } - end - - return _test_cases - end - def filter_gdb_test_report( report, test_case, filename ) lines = report.split( "\n" ) diff --git a/lib/ceedling/objects.yml b/lib/ceedling/objects.yml index 9e271ff5..04e4e3f7 100644 --- a/lib/ceedling/objects.yml +++ b/lib/ceedling/objects.yml @@ -40,7 +40,7 @@ file_path_collection_utils: compose: - file_wrapper -unity_utils: +test_runner_manager: compose: - configurator @@ -213,7 +213,7 @@ generator: - loginator - plugin_manager - file_wrapper - - unity_utils + - test_runner_manager - generator_test_results_backtrace generator_helper: @@ -324,7 +324,7 @@ test_invoker_helper: - file_path_utils - file_wrapper - generator - - unity_utils + - test_runner_manager release_invoker: compose: diff --git a/lib/ceedling/plugin_reportinator.rb b/lib/ceedling/plugin_reportinator.rb index 31b332c4..7c1c23de 100644 --- a/lib/ceedling/plugin_reportinator.rb +++ b/lib/ceedling/plugin_reportinator.rb @@ -37,6 +37,51 @@ def generate_heading(message) return @reportinator.generate_heading(message) end + ## + ## Sample Test Results Output File (YAML) + ## ====================================== + ## + ## TestUsartModel.fail: + ## --- + ## :source: + ## :file: test/TestUsartModel.c + ## :dirname: test + ## :basename: TestUsartModel.c + ## :successes: + ## - :test: testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting + ## :line: 24 + ## :message: '' + ## :unity_test_time: 0 + ## - :test: testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately + ## :line: 49 + ## :message: '' + ## :unity_test_time: 0 + ## - :test: testShouldReturnErrorMessageUponInvalidTemperatureValue + ## :line: 55 + ## :message: '' + ## :unity_test_time: 0 + ## - :test: testShouldReturnWakeupMessage + ## :line: 61 + ## :message: '' + ## :unity_test_time: 0 + ## :failures: + ## - :test: testFail + ## :line: 39 + ## :message: Expected 2 Was 3 + ## :unity_test_time: 0 + ## :ignores: + ## - :test: testIgnore + ## :line: 34 + ## :message: '' + ## :unity_test_time: 0 + ## :counts: + ## :total: 6 + ## :passed: 4 + ## :failed: 1 + ## :ignored: 1 + ## :stdout: [] + ## :time: 0.006512000225484371 + def assemble_test_results(results_list, options={:boom => false}) aggregated_results = new_results() @@ -53,10 +98,11 @@ def run_test_results_report(hash, verbosity=Verbosity::NORMAL, &block) raise CeedlingException.new( "No test results report template has been set." ) end - run_report( @test_results_template, - hash, - verbosity, - &block + run_report( + @test_results_template, + hash, + verbosity, + &block ) end diff --git a/lib/ceedling/plugin_reportinator_helper.rb b/lib/ceedling/plugin_reportinator_helper.rb index b25f0f25..557f9166 100644 --- a/lib/ceedling/plugin_reportinator_helper.rb +++ b/lib/ceedling/plugin_reportinator_helper.rb @@ -75,7 +75,7 @@ def process_results(aggregate, results) end def run_report(template, hash, verbosity) - output = ERB.new( template, trim_mode: "%<>" ) + output = ERB.new( template, trim_mode: "%<>-" ) # Run the report template and log result with no log level heading @loginator.log( output.result(binding()), verbosity, LogLabels::NONE ) diff --git a/lib/ceedling/setupinator.rb b/lib/ceedling/setupinator.rb index abcd18d2..fd1fe4ae 100644 --- a/lib/ceedling/setupinator.rb +++ b/lib/ceedling/setupinator.rb @@ -65,7 +65,7 @@ def do_setup( app_cfg ) @ceedling[:plugin_reportinator].set_system_objects( @ceedling ) # Process options for additional test runner #defines and test runner command line arguments - @ceedling[:unity_utils].process_test_runner_build_options() + @ceedling[:test_runner_manager].validate_and_configure_options() # Logging set up @ceedling[:loginator].set_logfile( form_log_filepath( log_filepath ) ) diff --git a/lib/ceedling/test_invoker_helper.rb b/lib/ceedling/test_invoker_helper.rb index 00260292..f92820fd 100644 --- a/lib/ceedling/test_invoker_helper.rb +++ b/lib/ceedling/test_invoker_helper.rb @@ -22,7 +22,7 @@ class TestInvokerHelper :file_path_utils, :file_wrapper, :generator, - :unity_utils + :test_runner_manager def setup # Alias for brevity @@ -105,7 +105,7 @@ def compile_defines(context:, filepath:) defines += @defineinator.defines( topkey:CEXCEPTION_SYM, subkey: :defines ) # Injected defines (based on other settings) - defines += @unity_utils.grab_additional_defines_based_on_configuration + defines += @test_runner_manager.collect_defines return defines.uniq end diff --git a/lib/ceedling/test_runner_manager.rb b/lib/ceedling/test_runner_manager.rb new file mode 100644 index 00000000..af7c0be8 --- /dev/null +++ b/lib/ceedling/test_runner_manager.rb @@ -0,0 +1,69 @@ +# ========================================================================= +# Ceedling - Test-Centered Build System for C +# ThrowTheSwitch.org +# Copyright (c) 2010-24 Mike Karlesky, Mark VanderVoord, & Greg Williams +# SPDX-License-Identifier: MIT +# ========================================================================= + +require 'ceedling/exceptions' + +class TestRunnerManager + + constructor :configurator + + def setup + @test_case_incl = nil + @test_case_excl = nil + @test_runner_defines = [] + end + + # Return test case arguments (empty if not set) + def collect_cmdline_args() + return [ @test_case_incl, @test_case_excl ].compact() + end + + def validate_and_configure_options() + # Blow up immediately if things aren't right + return if !validated_and_configured?() + + @test_runner_defines << 'UNITY_USE_COMMAND_LINE_ARGS' + + if !@configurator.include_test_case.empty? + @test_case_incl = "-f #{@configurator.include_test_case}" + end + + if !@configurator.exclude_test_case.empty? + @test_case_excl = "-x #{@configurator.exclude_test_case}" + end + end + + # Return ['UNITY_USE_COMMAND_LINE_ARGS'] #define required by Unity to enable cmd line arguments + def collect_defines() + return @test_runner_defines + end + + ### Private ### + + private + + # Raise exception if lacking support for test case matching + def validated_and_configured?() + # Command line arguments configured + cmdline_args = @configurator.test_runner_cmdline_args + + # Test case filters in use + test_case_filters = (!@configurator.include_test_case.nil? && !@configurator.include_test_case.empty?) || + (!@configurator.exclude_test_case.nil? && !@configurator.exclude_test_case.empty?) + + # Test case filters are in use but test runner command line arguments are not enabled + if test_case_filters and !cmdline_args + # Blow up if filters are in use but test runner command line arguments are not enabled + msg = 'Unity test case filters cannot be used as configured. ' + + 'Enable :test_runner ↳ :cmdline_args in your project configuration.' + + raise CeedlingException.new( msg ) + end + + return cmdline_args + end +end diff --git a/lib/ceedling/unity_utils.rb b/lib/ceedling/unity_utils.rb deleted file mode 100644 index e66981b8..00000000 --- a/lib/ceedling/unity_utils.rb +++ /dev/null @@ -1,119 +0,0 @@ -# ========================================================================= -# Ceedling - Test-Centered Build System for C -# ThrowTheSwitch.org -# Copyright (c) 2010-24 Mike Karlesky, Mark VanderVoord, & Greg Williams -# SPDX-License-Identifier: MIT -# ========================================================================= - -require 'ceedling/exceptions' - -# The Unity utils class, -# Store functions to enable test execution of single test case under test file -# and additional warning definitions -class UnityUtils - - constructor :configurator - - def setup - @test_case_incl = nil - @test_case_excl = nil - @test_runner_defines = [] - - # Refering to Unity implementation of the parser implemented in the unity.c : - # - # case 'l': /* list tests */ - # case 'f': /* filter tests with name including this string */ - # case 'q': /* quiet */ - # case 'v': /* verbose */ - # case 'x': /* exclude tests with name including this string */ - @arg_option_map = - { - :test_case => 'f', - :list_test_cases => 'l', - :run_tests_verbose => 'v', - :exclude_test_case => 'x' - } - end - - # Create test runner args which can be passed to executable test file as - # filter to execute one test case from test file - # - # @param [String, #argument] argument passed after test file name - # e.g.: ceedling test:: - # @param [String, #option] one of the supported by unity arguments. - # At current moment only "test_case_name" to - # run single test - # - # @return String - empty if cmdline_args is not set - # In other way properly formated command line for Unity - def additional_test_run_args(argument, option) - # Confirm wherever cmdline_args is set to true - # and parsing arguments under generated test runner in Unity is enabled - # and passed argument is not nil - - return nil if argument.nil? - - if !@arg_option_map.key?(option) - keys = @arg_option_map.keys.map{|key| ':' + key.to_s}.join(', ') - error = "option argument must be a known key {#{keys}}" - raise TypeError.new( error ) - end - - return " -#{@arg_option_map[option]} #{argument}" - end - - # Return test case arguments - # - # @return [String] formatted arguments for test file - def collect_test_runner_additional_args() - return [ @test_case_incl, @test_case_excl ].compact() - end - - # Parse passed by user arguments - def process_test_runner_build_options() - # Blow up immediately if things aren't right - return if !test_runner_cmdline_args_configured?() - - @test_runner_defines << 'UNITY_USE_COMMAND_LINE_ARGS' - - if !@configurator.include_test_case.empty? - @test_case_incl = additional_test_run_args( @configurator.include_test_case, :test_case ) - end - - if !@configurator.exclude_test_case.empty? - @test_case_excl = additional_test_run_args( @configurator.exclude_test_case, :exclude_test_case ) - end - end - - # Return UNITY_USE_COMMAND_LINE_ARGS define required by Unity to compile executable with enabled cmd line arguments - # - # @return [Array] - empty if cmdline_args is not set - def grab_additional_defines_based_on_configuration() - return @test_runner_defines - end - - ### Private ### - - private - - # Raise exception if lacking support for test case matching - def test_runner_cmdline_args_configured?() - # Command line arguments configured - cmdline_args = @configurator.test_runner_cmdline_args - - # Test case filters in use - test_case_filters = (!@configurator.include_test_case.nil? && !@configurator.include_test_case.empty?) || - (!@configurator.exclude_test_case.nil? && !@configurator.exclude_test_case.empty?) - - # Test case filters are in use but test runner command line arguments are not enabled - if test_case_filters and !cmdline_args - # Blow up if filters are in use but test runner command line arguments are not enabled - msg = 'Unity test case filters cannot be used as configured. ' + - 'Enable :test_runner ↳ :cmdline_args in your project configuration.' - - raise CeedlingException.new( msg ) - end - - return cmdline_args - end -end