Skip to content

Commit cc1bd05

Browse files
ZippeyKeys12ZippeyKeys12
authored
[CN-Test-Gen] Allow controlling progress output (#751)
1 parent 1425590 commit cc1bd05

File tree

7 files changed

+64
-14
lines changed

7 files changed

+64
-14
lines changed

‎backend/cn/bin/main.ml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,7 @@ let run_tests
462462
null_in_every
463463
seed
464464
logging_level
465+
progress_level
465466
interactive
466467
until_timeout
467468
exit_fast
@@ -537,6 +538,7 @@ let run_tests
537538
null_in_every;
538539
seed;
539540
logging_level;
541+
progress_level;
540542
interactive;
541543
until_timeout;
542544
exit_fast;
@@ -998,6 +1000,17 @@ module Testing_flags = struct
9981000
& info [ "logging-level" ] ~doc)
9991001

10001002

1003+
let progress_level =
1004+
let doc =
1005+
"Set the level of detail for progress updates (0 = Quiet, 1 = Per function, 2 = \
1006+
Per test case)"
1007+
in
1008+
Arg.(
1009+
value
1010+
& opt (some int) TestGeneration.default_cfg.progress_level
1011+
& info [ "progress-level" ] ~doc)
1012+
1013+
10011014
let interactive =
10021015
let doc =
10031016
"Enable interactive features for testing, such as requesting more detailed logs"
@@ -1117,6 +1130,7 @@ let testing_cmd =
11171130
$ Testing_flags.null_in_every
11181131
$ Testing_flags.seed
11191132
$ Testing_flags.logging_level
1133+
$ Testing_flags.progress_level
11201134
$ Testing_flags.interactive
11211135
$ Testing_flags.until_timeout
11221136
$ Testing_flags.exit_fast

‎backend/cn/lib/testGeneration/specTests.ml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,10 @@ let compile_script ~(output_dir : string) ~(test_file : string) : Pp.document =
451451
|> Option.map (fun level -> [ "--logging-level"; string_of_int level ])
452452
|> Option.to_list
453453
|> List.flatten)
454+
@ (Config.has_progress_level ()
455+
|> Option.map (fun level -> [ "--progress-level"; string_of_int level ])
456+
|> Option.to_list
457+
|> List.flatten)
454458
@ (if Config.is_interactive () then
455459
[ "--interactive" ]
456460
else

‎backend/cn/lib/testGeneration/testGenConfig.ml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ type t =
99
null_in_every : int option;
1010
seed : string option;
1111
logging_level : int option;
12+
progress_level : int option;
1213
interactive : bool;
1314
until_timeout : int option;
1415
exit_fast : bool;
@@ -31,6 +32,7 @@ let default =
3132
null_in_every = None;
3233
seed = None;
3334
logging_level = None;
35+
progress_level = None;
3436
interactive = false;
3537
until_timeout = None;
3638
exit_fast = false;
@@ -65,6 +67,8 @@ let has_seed () = !instance.seed
6567

6668
let has_logging_level () = !instance.logging_level
6769

70+
let has_progress_level () = !instance.progress_level
71+
6872
let is_interactive () = !instance.interactive
6973

7074
let is_until_timeout () = !instance.until_timeout

‎backend/cn/lib/testGeneration/testGenConfig.mli

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ type t =
99
null_in_every : int option;
1010
seed : string option;
1111
logging_level : int option;
12+
progress_level : int option;
1213
interactive : bool;
1314
until_timeout : int option;
1415
exit_fast : bool;
@@ -42,6 +43,8 @@ val has_seed : unit -> string option
4243

4344
val has_logging_level : unit -> int option
4445

46+
val has_progress_level : unit -> int option
47+
4548
val is_interactive : unit -> bool
4649

4750
val is_until_timeout : unit -> int option

‎runtime/libcn/include/cn-testing/test.h

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,13 @@
66
#include <cn-testing/uniform.h>
77
#include <cn-testing/result.h>
88

9-
typedef enum cn_test_result cn_test_case_fn(int);
9+
enum cn_test_gen_progress {
10+
CN_TEST_GEN_PROGRESS_NONE = 0,
11+
CN_TEST_GEN_PROGRESS_FINAL = 1,
12+
CN_TEST_GEN_PROGRESS_ALL = 2
13+
};
14+
15+
typedef enum cn_test_result cn_test_case_fn(enum cn_test_gen_progress);
1016

1117
void cn_register_test_case(char* suite, char* name, cn_test_case_fn* func);
1218

@@ -38,26 +44,32 @@ void print_test_info(char* suite, char* name, int tests, int discards);
3844
longjmp(buf_##Name, mode); \
3945
} \
4046
\
41-
enum cn_test_result cn_test_##Name (int printing) { \
47+
enum cn_test_result cn_test_##Name (enum cn_test_gen_progress progress_level) { \
4248
cn_gen_rand_checkpoint checkpoint = cn_gen_rand_save(); \
4349
int i = 0, d = 0; \
4450
set_cn_failure_cb(&cn_test_##Name##_fail); \
4551
switch (setjmp(buf_##Name)) { \
4652
case CN_FAILURE_ASSERT: \
4753
case CN_FAILURE_CHECK_OWNERSHIP: \
4854
case CN_FAILURE_OWNERSHIP_LEAK: \
55+
if (progress_level == CN_TEST_GEN_PROGRESS_FINAL) { \
56+
print_test_info(#Suite, #Name, i, d); \
57+
} \
4958
return CN_TEST_FAIL; \
5059
case CN_FAILURE_ALLOC: \
5160
cn_gen_rand_replace(checkpoint); \
5261
d++; \
5362
break; \
5463
} \
5564
for (; i < Samples; i++) { \
56-
if (printing) { \
65+
if (progress_level == CN_TEST_GEN_PROGRESS_ALL) { \
5766
printf("\r"); \
5867
print_test_info(#Suite, #Name, i, d); \
5968
} \
6069
if (d == 10 * Samples) { \
70+
if (progress_level == CN_TEST_GEN_PROGRESS_FINAL) { \
71+
print_test_info(#Suite, #Name, i, d); \
72+
} \
6173
return CN_TEST_GEN_FAIL; \
6274
} \
6375
size_t sz = cn_gen_uniform_cn_bits_u16(cn_gen_get_max_size())->val + 1; \
@@ -77,8 +89,10 @@ void print_test_info(char* suite, char* name, int tests, int discards);
7789
cn_gen_rand_replace(checkpoint); \
7890
} \
7991
\
80-
if (printing) { \
81-
printf("\r"); \
92+
if (progress_level != CN_TEST_GEN_PROGRESS_NONE) { \
93+
if (progress_level == CN_TEST_GEN_PROGRESS_ALL) { \
94+
printf("\r"); \
95+
} \
8296
print_test_info(#Suite, #Name, i, d); \
8397
} \
8498
return CN_TEST_PASS; \

‎runtime/libcn/src/cn-testing/test.c

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ int cn_test_main(int argc, char* argv[]) {
5757
set_cn_logging_level(CN_LOGGING_NONE);
5858

5959
cn_gen_srand(cn_gen_get_milliseconds());
60+
enum cn_test_gen_progress progress_level = CN_TEST_GEN_PROGRESS_ALL;
6061
uint64_t seed = cn_gen_rand();
6162
int interactive = 0;
6263
enum cn_logging_level logging_level = CN_LOGGING_ERROR;
@@ -77,6 +78,10 @@ int cn_test_main(int argc, char* argv[]) {
7778
logging_level = strtol(argv[i + 1], NULL, 10);
7879
i++;
7980
}
81+
else if (strcmp("--progress-level", arg) == 0) {
82+
progress_level = strtol(argv[i + 1], NULL, 10);
83+
i++;
84+
}
8085
else if (strcmp("--input-timeout", arg) == 0) {
8186
input_timeout = strtol(argv[i + 1], NULL, 10);
8287
i++;
@@ -140,13 +145,19 @@ int cn_test_main(int argc, char* argv[]) {
140145
}
141146

142147
struct cn_test_case* test_case = &test_cases[i];
143-
print_test_info(test_case->suite, test_case->name, 0, 0);
148+
if (progress_level == CN_TEST_GEN_PROGRESS_ALL) {
149+
print_test_info(test_case->suite, test_case->name, 0, 0);
150+
}
144151
checkpoints[i] = cn_gen_rand_save();
145152
cn_gen_set_input_timeout(input_timeout);
146-
enum cn_test_result result = test_case->func(1);
153+
enum cn_test_result result = test_case->func(progress_level);
147154
if (!(results[i] == CN_TEST_PASS && result == CN_TEST_GEN_FAIL)) {
148155
results[i] = result;
149156
}
157+
if (progress_level == CN_TEST_GEN_PROGRESS_NONE) {
158+
continue;
159+
}
160+
150161
printf("\n");
151162
switch (result) {
152163
case CN_TEST_PASS:
@@ -157,7 +168,7 @@ int cn_test_main(int argc, char* argv[]) {
157168
set_cn_logging_level(logging_level);
158169
cn_gen_rand_restore(checkpoints[i]);
159170
cn_gen_set_input_timeout(0);
160-
test_case->func(0);
171+
test_case->func(CN_TEST_GEN_PROGRESS_NONE);
161172
set_cn_logging_level(CN_LOGGING_NONE);
162173
printf("\n\n");
163174
break;

‎tests/run-cn-test-gen.sh

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,26 +30,26 @@ function separator() {
3030
CONFIGS=("--coverage" "--sized-null" "--random-size-splits" "--random-size-splits --allowed-size-split-backtracks=10")
3131

3232
# For each configuration
33-
for CONFIG in ${CONFIGS[@]}; do
33+
for CONFIG in "${CONFIGS[@]}"; do
3434
separator
3535
echo "Running CI with CLI config \"$CONFIG\""
3636
separator
3737

38-
CONFIG="$CONFIG --allowed-depth-failures=100 --input-timeout=1000"
38+
FULL_CONFIG="$CONFIG --allowed-depth-failures=100 --input-timeout=1000 --progress-level=1"
3939

4040
# Test each `*.c` file
4141
for TEST in $FILES; do
4242
CLEANUP="rm -rf test/* run_tests.sh;separator"
4343

4444
# Run passing tests
4545
if [[ $TEST == *.pass.c ]]; then
46-
$CN test "$TEST" --output-dir="test" $CONFIG
46+
$CN test "$TEST" --output-dir="test" $FULL_CONFIG
4747
RET=$?
4848
if [[ "$RET" != 0 ]]; then
4949
echo
5050
echo "$TEST -- Tests failed unexpectedly"
5151
NUM_FAILED=$(($NUM_FAILED + 1))
52-
FAILED="$FAILED $TEST"
52+
FAILED="$FAILED $TEST($CONFIG)"
5353
eval "$CLEANUP"
5454
continue
5555
else
@@ -60,13 +60,13 @@ for CONFIG in ${CONFIGS[@]}; do
6060

6161
# Run failing tests
6262
if [[ $TEST == *.fail.c ]]; then
63-
$CN test "$TEST" --output-dir="test" $CONFIG
63+
$CN test "$TEST" --output-dir="test" $FULL_CONFIG
6464
RET=$?
6565
if [[ "$RET" = 0 ]]; then
6666
echo
6767
echo "$TEST -- Tests passed unexpectedly"
6868
NUM_FAILED=$(($NUM_FAILED + 1))
69-
FAILED="$FAILED $TEST"
69+
FAILED="$FAILED $TEST($CONFIG)"
7070
eval "$CLEANUP"
7171
continue
7272
else

0 commit comments

Comments
 (0)