Add support for randomizing test execution order

This commit reintroduces the option to shuffle the test execution order
into the test runner. This has been tested with the temp_sensor example
project in Ceedling. Unit tests have also been successfully executed.

Signed-off-by: James Raphael Tiovalen <[email protected]>

Author: jamestiotio

auto/generate_test_runner.rb

@@ -47,7 +47,9 @@ def self.default_options
       use_param_tests: false,
       use_system_files: true,
       include_extensions: '(?:hpp|hh|H|h)',
-      source_extensions: '(?:cpp|cc|ino|C|c)'
+      source_extensions: '(?:cpp|cc|ino|C|c)',
+      shuffle_tests: false,
+      rng_seed: 0
     }
   end
 
@@ -90,6 +92,7 @@ def run(input_file, output_file, options = nil)
   def generate(input_file, output_file, tests, used_mocks, testfile_includes)
     File.open(output_file, 'w') do |output|
       create_header(output, used_mocks, testfile_includes)
+      create_run_test_params_struct(output)
       create_externs(output, tests, used_mocks)
       create_mock_management(output, used_mocks)
       create_setup(output)
@@ -99,6 +102,7 @@ def generate(input_file, output_file, tests, used_mocks, testfile_includes)
       create_reset(output)
       create_run_test(output) unless tests.empty?
       create_args_wrappers(output, tests)
+      create_shuffle_tests(output) if @options[:shuffle_tests]
       create_main(output, input_file, tests, used_mocks)
     end
 
@@ -231,16 +235,40 @@ def find_setup_and_teardown(source)
     @options[:has_suite_teardown] ||= (source =~ /int\s+suiteTearDown\s*\(int\s+([a-zA-Z0-9_])+\s*\)/)
   end
 
+  def count_tests(tests)
+    if @options[:use_param_tests]
+      idx = 0
+      tests.each do |test|
+        if (test[:args].nil? || test[:args].empty?)
+          idx += 1
+        else
+          test[:args].each do |args|
+            idx += 1
+          end
+        end
+      end
+      return idx
+    else
+      return tests.size
+    end
+  end
+
   def create_header(output, mocks, testfile_includes = [])
     output.puts('/* AUTOGENERATED FILE. DO NOT EDIT. */')
     output.puts("\n/*=======Automagically Detected Files To Include=====*/")
     output.puts('extern "C" {') if @options[:externcincludes]
+    if @options[:shuffle_tests]
+      output.puts('#include <stdlib.h>')
+      if @options[:rng_seed] == 0
+        output.puts('#include <time.h>')
+      end
+    end
     output.puts("#include \"#{@options[:framework]}.h\"")
     output.puts('#include "cmock.h"') unless mocks.empty?
     output.puts('}') if @options[:externcincludes]
     if @options[:defines] && !@options[:defines].empty?
       output.puts("/* injected defines for unity settings, etc */")
-      @options[:defines].each do |d| 
+      @options[:defines].each do |d|
         def_only = d.match(/(\w+).*/)[1]
         output.puts("#ifndef #{def_only}\n#define #{d}\n#endif /* #{def_only} */")
       end
@@ -270,6 +298,16 @@ def create_header(output, mocks, testfile_includes = [])
     output.puts('char* GlobalOrderError;')
   end
 
+  def create_run_test_params_struct(output)
+    output.puts("\n/*=======Structure Used By Test Runner=====*/")
+    output.puts('struct UnityRunTestParameters')
+    output.puts('{')
+    output.puts('  UnityTestFunction func;')
+    output.puts('  const char* name;')
+    output.puts('  UNITY_LINE_TYPE line_num;')
+    output.puts('};')
+  end
+
   def create_externs(output, tests, _mocks)
     output.puts("\n/*=======External Functions This Runner Calls=====*/")
     output.puts("extern void #{@options[:setup_name]}(void);")
@@ -392,6 +430,22 @@ def create_args_wrappers(output, tests)
     end
   end
 
+  def create_shuffle_tests(output)
+    output.puts("\n/*=======Shuffle Test Order=====*/")
+    output.puts('static void shuffleTests(struct UnityRunTestParameters run_test_params_arr[], int num_of_tests)')
+    output.puts('{')
+
+    # Use Fisher-Yates shuffle algorithm
+    output.puts('  for (int i = num_of_tests - 1; i > 0; i--)')
+    output.puts('  {')
+    output.puts('    int j = rand() % (i + 1);')
+    output.puts('    struct UnityRunTestParameters temp = run_test_params_arr[i];')
+    output.puts('    run_test_params_arr[i] = run_test_params_arr[j];')
+    output.puts('    run_test_params_arr[j] = temp;')
+    output.puts('  }')
+    output.puts('}')
+  end
+
   def create_main(output, filename, tests, used_mocks)
     output.puts("\n/*=======MAIN=====*/")
     main_name = @options[:main_name].to_sym == :auto ? "main_#{filename.gsub('.c', '')}" : (@options[:main_name]).to_s
@@ -437,18 +491,46 @@ def create_main(output, filename, tests, used_mocks)
     else
       output.puts("  UnityBegin(\"#{filename.gsub(/\\/, '\\\\\\')}\");")
     end
+    if @options[:shuffle_tests]
+      output.puts
+      if @options[:rng_seed] == 0
+        output.puts('  srand(time(NULL));')
+      else
+        output.puts("  srand(#{@options[:rng_seed]});")
+      end
+    end
+    output.puts
+    output.puts("  int number_of_tests = #{count_tests(tests)};")
+    output.puts('  struct UnityRunTestParameters run_test_params_arr[number_of_tests];')
+    output.puts
+    idx = 0
     tests.each do |test|
       if (!@options[:use_param_tests]) || test[:args].nil? || test[:args].empty?
-        output.puts("  run_test(#{test[:test]}, \"#{test[:test]}\", #{test[:line_number]});")
+        output.puts("  run_test_params_arr[#{idx}].func = #{test[:test]};")
+        output.puts("  run_test_params_arr[#{idx}].name = \"#{test[:test]}\";")
+        output.puts("  run_test_params_arr[#{idx}].line_num = #{test[:line_number]};")
+        idx += 1
       else
-        test[:args].each.with_index(1) do |args, idx|
-          wrapper = "runner_args#{idx}_#{test[:test]}"
+        test[:args].each.with_index(1) do |args, arg_idx|
+          wrapper = "runner_args#{arg_idx}_#{test[:test]}"
           testname = "#{test[:test]}(#{args})".dump
-          output.puts("  run_test(#{wrapper}, #{testname}, #{test[:line_number]});")
+          output.puts("  run_test_params_arr[#{idx}].func = #{wrapper};")
+          output.puts("  run_test_params_arr[#{idx}].name = #{testname};")
+          output.puts("  run_test_params_arr[#{idx}].line_num = #{test[:line_number]};")
+          idx += 1
         end
       end
     end
     output.puts
+    if @options[:shuffle_tests]
+      output.puts('  shuffleTests(run_test_params_arr, number_of_tests);')
+      output.puts
+    end
+    output.puts('  for (int i = 0; i < number_of_tests; i++)')
+    output.puts('  {')
+    output.puts('    run_test(run_test_params_arr[i].func, run_test_params_arr[i].name, run_test_params_arr[i].line_num);')
+    output.puts('  }')
+    output.puts
     output.puts('  CMock_Guts_MemFreeFinal();') unless used_mocks.empty?
     if @options[:has_suite_teardown]
       if @options[:omit_begin_end]
@@ -534,7 +616,9 @@ def create_h_file(output, filename, tests, testfile_includes, used_mocks)
           '    --suite_teardown=""   - code to execute for teardown of entire suite',
           '    --use_param_tests=1   - enable parameterized tests (disabled by default)',
           '    --omit_begin_end=1    - omit calls to UnityBegin and UnityEnd (disabled by default)',
-          '    --header_file=""      - path/name of test header file to generate too'].join("\n")
+          '    --header_file=""      - path/name of test header file to generate too',
+          '    --shuffle_tests=1     - enable shuffling of the test execution order (disabled by default)',
+          '    --rng_seed=1          - seed value for randomization of test execution order'].join("\n")
     exit 1
   end
 

docs/UnityHelperScriptsGuide.md

@@ -254,6 +254,18 @@ Provide any `cdecl` for the `main()` test function. Is empty by default.
 If `true`, the `UnityBegin` and `UnityEnd` function will not be called for
 Unity test state setup and cleanup.
 
+##### `:shuffle_tests`
+
+If `true`, the test execution order will be shuffled. Is `false` by default.
+
+##### `:rng_seed`
+
+If set to some positive integer value, said value will be used as the seed value passed
+to the `srand` function. Otherwise, if not set to any value, `time(NULL)` will be used
+as the seed value.
+
+Only applicable if `:shuffle_tests` is set to `true`.
+
 #### Parameterized tests provided macros
 
 Unity provides support for few param tests generators, that can be combined