Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for randomizing test execution order #714

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add support for randomizing test execution order #714

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 91 additions & 7 deletions auto/generate_test_runner.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ def self.default_options
use_param_tests: false,
use_system_files: true,
include_extensions: '(?:hpp|hh|H|h)',
source_extensions: '(?:cpp|cc|ino|C|c)'
source_extensions: '(?:cpp|cc|ino|C|c)',
shuffle_tests: false,
rng_seed: 0
}
end

Expand Down Expand Up @@ -90,6 +92,7 @@ def run(input_file, output_file, options = nil)
def generate(input_file, output_file, tests, used_mocks, testfile_includes)
File.open(output_file, 'w') do |output|
create_header(output, used_mocks, testfile_includes)
create_run_test_params_struct(output)
create_externs(output, tests, used_mocks)
create_mock_management(output, used_mocks)
create_setup(output)
Expand All @@ -99,6 +102,7 @@ def generate(input_file, output_file, tests, used_mocks, testfile_includes)
create_reset(output)
create_run_test(output) unless tests.empty?
create_args_wrappers(output, tests)
create_shuffle_tests(output) if @options[:shuffle_tests]
create_main(output, input_file, tests, used_mocks)
end

Expand Down Expand Up @@ -231,16 +235,40 @@ def find_setup_and_teardown(source)
@options[:has_suite_teardown] ||= (source =~ /int\s+suiteTearDown\s*\(int\s+([a-zA-Z0-9_])+\s*\)/)
end

def count_tests(tests)
if @options[:use_param_tests]
idx = 0
tests.each do |test|
if (test[:args].nil? || test[:args].empty?)
idx += 1
else
test[:args].each do |args|
idx += 1
end
end
end
return idx
else
return tests.size
end
end

def create_header(output, mocks, testfile_includes = [])
output.puts('/* AUTOGENERATED FILE. DO NOT EDIT. */')
output.puts("\n/*=======Automagically Detected Files To Include=====*/")
output.puts('extern "C" {') if @options[:externcincludes]
if @options[:shuffle_tests]
output.puts('#include <stdlib.h>')
if @options[:rng_seed] == 0
output.puts('#include <time.h>')
end
end
output.puts("#include \"#{@options[:framework]}.h\"")
output.puts('#include "cmock.h"') unless mocks.empty?
output.puts('}') if @options[:externcincludes]
if @options[:defines] && !@options[:defines].empty?
output.puts("/* injected defines for unity settings, etc */")
@options[:defines].each do |d|
@options[:defines].each do |d|
def_only = d.match(/(\w+).*/)[1]
output.puts("#ifndef #{def_only}\n#define #{d}\n#endif /* #{def_only} */")
end
Expand Down Expand Up @@ -270,6 +298,16 @@ def create_header(output, mocks, testfile_includes = [])
output.puts('char* GlobalOrderError;')
end

def create_run_test_params_struct(output)
output.puts("\n/*=======Structure Used By Test Runner=====*/")
output.puts('struct UnityRunTestParameters')
output.puts('{')
output.puts(' UnityTestFunction func;')
output.puts(' const char* name;')
output.puts(' UNITY_LINE_TYPE line_num;')
output.puts('};')
end

def create_externs(output, tests, _mocks)
output.puts("\n/*=======External Functions This Runner Calls=====*/")
output.puts("extern void #{@options[:setup_name]}(void);")
Expand Down Expand Up @@ -392,6 +430,22 @@ def create_args_wrappers(output, tests)
end
end

def create_shuffle_tests(output)
output.puts("\n/*=======Shuffle Test Order=====*/")
output.puts('static void shuffleTests(struct UnityRunTestParameters run_test_params_arr[], int num_of_tests)')
output.puts('{')

# Use Fisher-Yates shuffle algorithm
output.puts(' for (int i = num_of_tests - 1; i > 0; i--)')
output.puts(' {')
output.puts(' int j = rand() % (i + 1);')
output.puts(' struct UnityRunTestParameters temp = run_test_params_arr[i];')
output.puts(' run_test_params_arr[i] = run_test_params_arr[j];')
output.puts(' run_test_params_arr[j] = temp;')
output.puts(' }')
output.puts('}')
end

def create_main(output, filename, tests, used_mocks)
output.puts("\n/*=======MAIN=====*/")
main_name = @options[:main_name].to_sym == :auto ? "main_#{filename.gsub('.c', '')}" : (@options[:main_name]).to_s
Expand Down Expand Up @@ -437,18 +491,46 @@ def create_main(output, filename, tests, used_mocks)
else
output.puts(" UnityBegin(\"#{filename.gsub(/\\/, '\\\\\\')}\");")
end
if @options[:shuffle_tests]
output.puts
if @options[:rng_seed] == 0
output.puts(' srand(time(NULL));')
else
output.puts(" srand(#{@options[:rng_seed]});")
end
end
output.puts
output.puts(" int number_of_tests = #{count_tests(tests)};")
output.puts(' struct UnityRunTestParameters run_test_params_arr[number_of_tests];')
output.puts
idx = 0
tests.each do |test|
if (!@options[:use_param_tests]) || test[:args].nil? || test[:args].empty?
output.puts(" run_test(#{test[:test]}, \"#{test[:test]}\", #{test[:line_number]});")
output.puts(" run_test_params_arr[#{idx}].func = #{test[:test]};")
output.puts(" run_test_params_arr[#{idx}].name = \"#{test[:test]}\";")
output.puts(" run_test_params_arr[#{idx}].line_num = #{test[:line_number]};")
idx += 1
else
test[:args].each.with_index(1) do |args, idx|
wrapper = "runner_args#{idx}_#{test[:test]}"
test[:args].each.with_index(1) do |args, arg_idx|
wrapper = "runner_args#{arg_idx}_#{test[:test]}"
testname = "#{test[:test]}(#{args})".dump
output.puts(" run_test(#{wrapper}, #{testname}, #{test[:line_number]});")
output.puts(" run_test_params_arr[#{idx}].func = #{wrapper};")
output.puts(" run_test_params_arr[#{idx}].name = #{testname};")
output.puts(" run_test_params_arr[#{idx}].line_num = #{test[:line_number]};")
idx += 1
end
end
end
output.puts
if @options[:shuffle_tests]
output.puts(' shuffleTests(run_test_params_arr, number_of_tests);')
output.puts
end
output.puts(' for (int i = 0; i < number_of_tests; i++)')
output.puts(' {')
output.puts(' run_test(run_test_params_arr[i].func, run_test_params_arr[i].name, run_test_params_arr[i].line_num);')
output.puts(' }')
output.puts
output.puts(' CMock_Guts_MemFreeFinal();') unless used_mocks.empty?
if @options[:has_suite_teardown]
if @options[:omit_begin_end]
Expand Down Expand Up @@ -534,7 +616,9 @@ def create_h_file(output, filename, tests, testfile_includes, used_mocks)
' --suite_teardown="" - code to execute for teardown of entire suite',
' --use_param_tests=1 - enable parameterized tests (disabled by default)',
' --omit_begin_end=1 - omit calls to UnityBegin and UnityEnd (disabled by default)',
' --header_file="" - path/name of test header file to generate too'].join("\n")
' --header_file="" - path/name of test header file to generate too',
' --shuffle_tests=1 - enable shuffling of the test execution order (disabled by default)',
' --rng_seed=1 - seed value for randomization of test execution order'].join("\n")
exit 1
end

Expand Down
12 changes: 12 additions & 0 deletions docs/UnityHelperScriptsGuide.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,18 @@ Provide any `cdecl` for the `main()` test function. Is empty by default.
If `true`, the `UnityBegin` and `UnityEnd` function will not be called for
Unity test state setup and cleanup.

##### `:shuffle_tests`

If `true`, the test execution order will be shuffled. Is `false` by default.

##### `:rng_seed`

If set to some positive integer value, said value will be used as the seed value passed
to the `srand` function. Otherwise, if not set to any value, `time(NULL)` will be used
as the seed value.

Only applicable if `:shuffle_tests` is set to `true`.

#### Parameterized tests provided macros

Unity provides support for few param tests generators, that can be combined
Expand Down