add make comparison and make lexer-comparison tasks to help us co…

…mpare the full set of examples` output vs. a given reference. This is basically a 'system test' / 'acceptance test' **test level** that co-exists with the unit tests and integration tests in the tests/ directory: those tests are already partly leaning towards a 'system test' level and that is "polluting" the implied simplicity of unit tests...
GerHobbelt · Oct 30, 2017 · bd2a4ed · bd2a4ed
1 parent 0e3ee7b
commit bd2a4ed
Show file tree

Hide file tree

Showing 7 changed files with 58 additions and 22 deletions.
diff --git a/Makefile b/Makefile
@@ -78,6 +78,12 @@ web/content/assets/js/calculator.js: examples/calculator.jison build
 	$(JISON) examples/calculator.jison -o $@
 
 
+comparison: 
+	cd examples/ && make comparison
+
+lexer-comparison: build
+	cd packages/jison-lex && make comparison
+
 examples_directory: build
 	cd examples/ && make all
 
@@ -330,7 +336,7 @@ examples/parser-to-lexer-communication-test: build
 examples/parser-to-lexer-communication-test--profiling: build
 	cd examples/ && make parser-to-lexer-communication-test--profiling
 
-profiling: 
+profiling:
 	cd examples/ && make profiling
 
 examples/pascal: build
@@ -415,7 +421,7 @@ build:                                                                  \
 npm-install:
 	npm install
 
-npm-update: subpackages-npm-update 
+npm-update: subpackages-npm-update
 	ncu -a --packageFile=package.json
 
 prep_util_dir:
@@ -554,6 +560,7 @@ superclean: clean clean-site
 		prep subpackages-prep                                                       \
 		helpers-lib lex-parser jison-lex ebnf-parser json2jison jison2json          \
 		site preview deploy test web-examples examples examples-test                \
+		examples_directory comparison lexer-comparison                              \
 		error-handling-tests basic-tests github-issue-tests misc-tests              \
 		build npm-install                                                           \
 		subpackages                                                                 \
@@ -562,5 +569,5 @@ superclean: clean clean-site
 		git-tag subpackages-git-tag                                                 \
 		compile-site clean-site                                                     \
 		publish subpackages-publish                                                 \
-		npm-update subpackages-npm-update 
+		npm-update subpackages-npm-update
 
diff --git a/examples/Makefile b/examples/Makefile
@@ -34,11 +34,11 @@ GITHUB_ISSUE_TARGETS =                                          \
 			issue-344-BAD                                       \
 			issue-344-2-BAD                                     \
 			issue-348                                           \
-			issue-357-url-lexing								\
-			issue-360                                           
+			issue-357-url-lexing                                \
+			issue-360
 
 
-CODEGEN_TARGETS = 												\
+CODEGEN_TARGETS =                                               \
 		codegen-feature-tester-base                             \
 		codegen-feature-tester-base-debug                       \
 		codegen-feature-tester-base-try-catch                   \
@@ -66,7 +66,7 @@ CODEGEN_TARGETS = 												\
 		codegen-feature-tester-no-yysp                          \
 		codegen-feature-tester-no-yysstack                      \
 		codegen-feature-tester-no-yystack                       \
-		codegen-feature-tester-no-yytext                                    
+		codegen-feature-tester-no-yytext
 
 
 
@@ -83,7 +83,7 @@ BASIC_TARGETS =                                                 \
 			ccalc                                               \
 			classy                                              \
 			classy_ast                                          \
-			codegen-feature-tester                              \
+			$(CODEGEN_TARGETS)                                  \
 			comments                                            \
 			compiled_calc                                       \
 			dism                                                \
@@ -172,7 +172,7 @@ MISC_TARGETS =                                                  \
 			lr-but-not-lalr                                     \
 			mfcalc                                              \
 			no-prec-hack-needed                                 \
-			codegen-feature-tester                              \
+			$(CODEGEN_TARGETS)                                  \
 			nv_classy_ast                                       \
 			olmenu-proto2                                       \
 			parser-to-lexer-communication-test                  \
@@ -205,7 +205,7 @@ ALL_TARGETS =                                                   \
 			$(GITHUB_ISSUE_TARGETS)                             \
 			$(MISC_TARGETS)                                     \
 			$(ERROR_HANDLING_TARGETS)                           \
-			$(CODEGEN_TARGETS) 									\
+			$(CODEGEN_TARGETS)                                  \
 			ansic                                               \
 			btyacc-ansiC                                        \
 			btyacc-ansiC2                                       \
@@ -233,7 +233,7 @@ ALL_TARGETS =                                                   \
 
 all: $(ALL_TARGETS)
 
-.PHONY: all clean superclean error-handling-tests basic-tests github-issue-tests misc-tests $(ALL_TARGETS) 
+.PHONY: all clean superclean error-handling-tests basic-tests github-issue-tests misc-tests $(ALL_TARGETS) comparison
 
 
 clean:
@@ -540,10 +540,10 @@ btyacc-t2:
 error-only:
 	$(JISON) --main ./$@.y
 
-profiling: 															\
-			parser-to-lexer-communication-test						\
-			parser-to-lexer-communication-test-w-debug				\
-			parser-to-lexer-communication-test--profiling 			\
+profiling:                                                          \
+			parser-to-lexer-communication-test                      \
+			parser-to-lexer-communication-test-w-debug              \
+			parser-to-lexer-communication-test--profiling           \
 			parser-to-lexer-communication-test-w-debug--profiling
 
 parser-to-lexer-communication-test--profiling:
@@ -686,15 +686,17 @@ compiled_calc:
 
 CGT=codegen-feature-tester
 
-codegen-feature-tester:                                                     \
+comparison:                                                                 \
+		clean                                                               \
+		$(ALL_TARGETS)                                                      \
 		$(CODEGEN_TARGETS)                                                  \
 		reference-output/README.md
-	# only run a DIFF once we have generated all outputs: 
-	# that way we get to see all diffs at once and probably have a 
+	# only run a DIFF once we have generated all outputs:
+	# that way we get to see all diffs at once and probably have a
 	# faster turn-around cycle when we go and fix any diffs observed
 	# here.
-	-diff -C 1 -t --tabsize=4 -a -I ' generated by ' -B -E -Z -w ./reference-output/codegen-feature-tester/ ./output/codegen-feature-tester/    > ./codegen-feature-tester.diff-report.patch
-	diff -q    -t --tabsize=4 -a -I ' generated by ' -B -E -Z -w ./reference-output/codegen-feature-tester/ ./output/codegen-feature-tester/    
+	#-diff -C 1 -r -t --tabsize=4 -a -I ' generated by ' -B -E -Z -w ./reference-output/ ./output/    > ./comparison.diff-report.patch
+	diff -q -r -t --tabsize=4 -a -I ' generated by ' -B -E -Z -w -X comparison-diff-filter-extensions.txt ./reference-output/ ./output/
 
 
 codegen-feature-tester-base:

diff --git a/examples/comparison-diff-filter-extensions.txt b/examples/comparison-diff-filter-extensions.txt
@@ -0,0 +1,2 @@
+*.txt
+*.log
diff --git a/examples/reference-output/README.md b/examples/reference-output/README.md
@@ -3,7 +3,7 @@
 Reference directory for tests such as
 
 ```
-make codegen-feature-tester
+make comparison
 ```
 
 which test whether jison generates proper code for parsers with various features enabled/disabled.
diff --git a/packages/jison-lex/Makefile b/packages/jison-lex/Makefile
@@ -140,6 +140,20 @@ examples_with_custom_lexer:
 	$(LEX) examples/with_custom_lexer.jison -o examples/output/ -x
 
 
+
+comparison:                                                                 \
+		clean                                                               \
+		examples                                                            \
+		examples/reference-output/README.md
+	# only run a DIFF once we have generated all outputs:
+	# that way we get to see all diffs at once and probably have a
+	# faster turn-around cycle when we go and fix any diffs observed
+	# here.
+	#-diff -C 1 -r -t --tabsize=4 -a -I ' generated by ' -B -E -Z -w examples/reference-output/ examples/output/    > ./comparison.diff-report.patch
+	diff -q -r -t --tabsize=4 -a -I ' generated by ' -B -E -Z -w -X examples/comparison-diff-filter-extensions.txt examples/reference-output/ examples/output/
+
+
+
 # increment the XXX <prelease> number in the package.json file: version <major>.<minor>.<patch>-<prelease>
 bump:
 	cd ../../ && node __patch_version_in_js.js
@@ -167,5 +181,5 @@ superclean: clean
 
 
 
-.PHONY: all prep npm-install build test examples clean superclean bump git-tag publish example-lex example-include examples_basic2_lex examples_basic_lex examples_c99 examples_ccalc_lex examples_classy examples_codegen_feature_tester_base examples_comments examples_compiled_calc_parse examples_faking examples_floop examples_handlebars examples_issue_url_lexing examples_issue_x1 examples_issue_x2 examples_lex_grammar examples_lexer_comm_debug examples_pascal examples_regex examples_semwhitespace examples_tikiwikiparser examples_unicode2 examples_unicode examples_with_custom_lexer examples_with_includes
+.PHONY: all prep npm-install build test examples clean superclean bump git-tag publish example-lex example-include examples_basic2_lex examples_basic_lex examples_c99 examples_ccalc_lex examples_classy examples_codegen_feature_tester_base examples_comments examples_compiled_calc_parse examples_faking examples_floop examples_handlebars examples_issue_url_lexing examples_issue_x1 examples_issue_x2 examples_lex_grammar examples_lexer_comm_debug examples_pascal examples_regex examples_semwhitespace examples_tikiwikiparser examples_unicode2 examples_unicode examples_with_custom_lexer examples_with_includes comparison
 
diff --git a/packages/jison-lex/examples/comparison-diff-filter-extensions.txt b/packages/jison-lex/examples/comparison-diff-filter-extensions.txt
@@ -0,0 +1,2 @@
+*.txt
+*.log
diff --git a/packages/jison-lex/examples/reference-output/README.md b/packages/jison-lex/examples/reference-output/README.md
@@ -0,0 +1,9 @@
+# Reference directory for generated code comparisons
+
+Reference directory for tests such as
+
+```
+make comparison
+```
+
+which test whether jison-lex generates proper code for lexers with various features enabled/disabled.