line ending fixes

.gitignore

@@ -15,3 +15,4 @@ sdlfmt-test-*
 *.obj
 *.lst
 test/checkfmt
+test/actual

source/sdlfmt.d

@@ -122,7 +122,7 @@ string format(SdlFmtConfig config, scope const(char)[] sdl, string filename = ""
 
 	bool isComment(scope const(char)[] sdl)
 	{
-		if (sdl.startsWith("//", "--"))
+		if (sdl.startsWith("//", "--", "#"))
 			return true;
 		else if (sdl.startsWith("/*"))
 			return sdl.endsWith("*/") && !sdl[0 .. $ - 2].canFind("*/");
@@ -167,6 +167,27 @@ string format(SdlFmtConfig config, scope const(char)[] sdl, string filename = ""
 		neededNewLines ~= lastNewLineIndex;
 	}
 
+	void appendFixedNewlines(T)(T text)
+	{
+		import std.uni : lineSep, paraSep;
+
+		foreach (line; text.lineSplitter!(KeepTerminator.yes))
+		{
+			if (line.endsWith("\r\n"))
+			{
+				ret ~= line[0 .. $ - 2];
+				ret ~= config.lineEnding;
+			}
+			else if (line.endsWith('\r', '\n', '\v', '\f', '\x85'))
+			{
+				ret ~= line[0 .. $ - 1];
+				ret ~= config.lineEnding;
+			}
+			else
+				ret ~= line;
+		}
+	}
+
 	foreach (token; lexSDLang(sdl, filename))
 	{
 		final switch (token.type)
@@ -182,8 +203,8 @@ string format(SdlFmtConfig config, scope const(char)[] sdl, string filename = ""
 			}
 			else
 			{
-				ret ~= token.whitespacePrefix;
-				ret ~= token.text;
+				appendFixedNewlines(token.whitespacePrefix);
+				appendFixedNewlines(token.text);
 			}
 			break;
 		case TokenType.eof: break;
@@ -199,7 +220,7 @@ string format(SdlFmtConfig config, scope const(char)[] sdl, string filename = ""
 			break;
 		case TokenType.semicolon:
 			putIndentIfNeeded();
-			ret ~= token.text;
+			ret ~= ";";
 			multiLineIndent = 0;
 			break;
 		case TokenType.blockClose:
@@ -220,7 +241,7 @@ string format(SdlFmtConfig config, scope const(char)[] sdl, string filename = ""
 			else
 				putIndentIfNeeded();
 
-			ret ~= token.text;
+			ret ~= "{";
 			queueNewLine = true;
 			break;
 		case TokenType.namespace:
@@ -243,8 +264,8 @@ string format(SdlFmtConfig config, scope const(char)[] sdl, string filename = ""
 			else
 				putIndentIfNeeded();
 
-			ret ~= token.text;
-			if (token.text.endsWith("\n", "\r"))
+			appendFixedNewlines(token.text);
+			if (ret.data.endsWith(config.lineEnding))
 				lastNewLineIndex = ret.data.length;
 			break;
 		case TokenType.null_:
@@ -261,11 +282,14 @@ string format(SdlFmtConfig config, scope const(char)[] sdl, string filename = ""
 			else
 				putIndentIfNeeded();
 
-			ret ~= token.text;
+			appendFixedNewlines(token.text);
 			break;
 		}
 	}
 
+	if (ret.data.length && !ret.data.endsWith(config.lineEnding))
+		ret ~= config.lineEnding;
+
 	auto data = ret.data;
 	if (!neededNewLines.length)
 		return data;

test/checkfmt.d

@@ -35,27 +35,32 @@ SdlFmtConfig deserializeConfig(string name)
 		{
 			name = name["tabs-".length .. $];
 			auto count = name.parse!int;
-			enforce(name.startsWith("-"), "invalid format, expected `tabs-N`, where N is an integer");
+			enforce(name.startsWith("-") || !name.length, "invalid format, expected `tabs-N`, where N is an integer");
 			ret.indent = '\t'.repeat(count).array;
 		}
 		else if (name.startsWith("spaces-"))
 		{
 			name = name["spaces-".length .. $];
 			auto count = name.parse!int;
-			enforce(name.startsWith("-"), "invalid format, expected `spaces-N`, where N is an integer");
+			enforce(name.startsWith("-") || !name.length, "invalid format, expected `spaces-N`, where N is an integer");
 			ret.indent = ' '.repeat(count).array;
 		}
 		else if (name.startsWith("tempws-"))
 		{
 			name = name["tempws-".length .. $];
 			ret.backslashTempIndent = name.parse!int;
-			enforce(name.startsWith("-"), "invalid format, expected `tempws-N`, where N is an integer");
+			enforce(name.startsWith("-") || !name.length, "invalid format, expected `tempws-N`, where N is an integer");
 		}
 		else if (name.startsWith("equals_ws"))
 		{
 			name = name["equals_ws".length .. $];
 			ret.whitespaceAroundEquals = true;
 		}
+		else if (name.startsWith("crlf"))
+		{
+			name = name["crlf".length .. $];
+			ret.lineEnding = "\r\n";
+		}
 		else if (name.startsWith("lf"))
 		{
 			name = name["lf".length .. $];
@@ -66,11 +71,6 @@ SdlFmtConfig deserializeConfig(string name)
 			name = name["cr".length .. $];
 			ret.lineEnding = "\r";
 		}
-		else if (name.startsWith("crlf"))
-		{
-			name = name["crlf".length .. $];
-			ret.lineEnding = "\r\n";
-		}
 		else
 			throw new Exception("Unknown format part, known parts: tabs-N, spaces-N, tempws-N, equals_ws, lf, cr, crlf");
 	}
@@ -80,10 +80,10 @@ SdlFmtConfig deserializeConfig(string name)
 
 int main(string[] args)
 {
+	import std.file : write;
+
 	if (args.length > 1 && args[1] == "generate")
 	{
-		import std.file : write;
-
 		enforce(args.length > 2, "Usage: " ~ args[0] ~ " generate [file] ([options])\nwhere options is the serialized options name (folder name in results)");
 		auto serializedOptions = args.length > 3 ? args[3] : "default";
 		auto options = deserializeConfig(serializedOptions);
@@ -116,10 +116,12 @@ int main(string[] args)
 		auto sourceName = buildPath("source", baseName(parts));
 		missingSourceFiles = missingSourceFiles.remove!(a => a == sourceName);
 
-		auto expected = config.format(sourceName.readText, sourceName);
-		auto actual = readText(test);
+		auto actual = config.format(sourceName.readText, sourceName);
+		auto expected = readText(test);
+
+		write(buildPath("actual", dirName(parts) ~ "-" ~ baseName(parts)), actual);
 
-		if (expected != actual)
+		if (actual != expected)
 		{
 			writeln("\x1B[1;31mFAIL\x1B[m");
 			fails++;

test/results/cr/many_things.sdl

@@ -0,0 +1 @@
+name "vibe-d"description "Event driven web and concurrency framework"homepage "https://vibed.org/"license "MIT"copyright "Copyright © 2012-2020 Sönke Ludwig"authors "Sönke Ludwig" "Mathias 'Geod24' Lang" "Etienne Cimon" "Martin Nowak" \		"Mihails 'Dicebot' Strasuns" "150 contributors total"systemDependencies "Optionally OpenSSL 1.1.x"dependency ":redis" version="*"dependency ":data" version="*"dependency ":inet" version="*"targetType "library"targetName "vibed"// NOTE: "lib" is a path with no D sources to work around an issue in DUB 1.0.0//       and below that results in the standard "source/" path to be added even//       if an explicit "sourcePaths" directive is given.sourcePaths "lib"sourceFiles "source/vibe/d.d" "source/vibe/vibe.d"x:ddoxFilterArgs "--unittest-examples" "--min-protection=Protected" \		"--ex" "vibe.core.drivers." "--ex" "vibe.internal." "--ex" "vibe.web.internal." \		"--ex" "diet.internal" "--ex" "stdx." "--ex" "eventcore.internal." "--ex" "eventcore.drivers." \		"--ex" "mir." "--ex" "openssl_version"configuration "unittest" {	targetPath "bin"	targetName "ut"	dflags "-preview=dip1000"	sourcePaths "source" "test/ut"	importPaths "source" "test"	stringImportPaths "test/testFiles"	foo {		bar {			""		}	}	versions "unitUnthreaded"	dependency "unit-threaded" version="*"	dependency "unit-threaded:autorunner" version="*"}// cool beans/* foo */foo {	// ok	bar {		// xd		x {			d		}	}}subPackage "utils"subPackage "data"subPackage "stream"

test/results/crlf/many_things.sdl

@@ -0,0 +1,64 @@
+name "vibe-d"
+description "Event driven web and concurrency framework"
+homepage "https://vibed.org/"
+
+license "MIT"
+copyright "Copyright © 2012-2020 Sönke Ludwig"
+authors "Sönke Ludwig" "Mathias 'Geod24' Lang" "Etienne Cimon" "Martin Nowak" \
+		"Mihails 'Dicebot' Strasuns" "150 contributors total"
+
+systemDependencies "Optionally OpenSSL 1.1.x"
+dependency ":redis" version="*"
+dependency ":data" version="*"
+dependency ":inet" version="*"
+
+targetType "library"
+targetName "vibed"
+
+// NOTE: "lib" is a path with no D sources to work around an issue in DUB 1.0.0
+//       and below that results in the standard "source/" path to be added even
+//       if an explicit "sourcePaths" directive is given.
+sourcePaths "lib"
+sourceFiles "source/vibe/d.d" "source/vibe/vibe.d"
+
+x:ddoxFilterArgs "--unittest-examples" "--min-protection=Protected" \
+		"--ex" "vibe.core.drivers." "--ex" "vibe.internal." "--ex" "vibe.web.internal." \
+		"--ex" "diet.internal" "--ex" "stdx." "--ex" "eventcore.internal." "--ex" "eventcore.drivers." \
+		"--ex" "mir." "--ex" "openssl_version"
+
+configuration "unittest" {
+	targetPath "bin"
+	targetName "ut"
+
+	dflags "-preview=dip1000"
+
+	sourcePaths "source" "test/ut"
+	importPaths "source" "test"
+	stringImportPaths "test/testFiles"
+
+	foo {
+		bar {
+			""
+		}
+	}
+	versions "unitUnthreaded"
+
+	dependency "unit-threaded" version="*"
+	dependency "unit-threaded:autorunner" version="*"
+}
+// cool beans
+
+/* foo */
+foo {
+	// ok
+	bar {
+		// xd
+		x {
+			d
+		}
+	}
+}
+
+subPackage "utils"
+subPackage "data"
+subPackage "stream"

test/results/default/comment_styles.sdl

@@ -0,0 +1,11 @@
+// C++ style
+
+/*
+C style multiline
+*/
+
+tag /*foo=true*/ bar=false
+
+# Shell style
+
+-- Lua style

test/results/default/data_types.sdl

@@ -0,0 +1,29 @@
+// Strings
+"String \"with escape support\""
+`String "without escape support"`
+
+// Numbers
+10 // 32-bit integer
+10L // 64-bit integer
+10.5 // 64-bit float (double)
+10.5d // 64-bit float
+10.5f // 32-bit float
+10.123BD // 128-bit decimal
+
+// Boolean and null values
+true // boolean true
+false // boolean false
+on // boolean true
+off // boolean false
+null // a null value
+
+// Date/time formats
+2015/12/06 12:00:00.000-UTC // Date/time value (UTC timezone)
+2015/12/06 12:00:00.000 // Date/time value (local time)
+2015/12/06 // Date value
+12:14:34 // Duration: 12 hours, 14 minutes, 34 seconds
+12:14:34.123 // 12 h, 14 min, 34 s, 123 ms
+2d:12:14:34 // 2 days, 12 h, 14 min, 34 s
+
+// Binary data (Base64 encoded)
+[sdf789GSfsb2+3324sf2]

test/results/default/syntax_details.sdl

@@ -0,0 +1,17 @@
+// Trailing semicolons are optional
+title "Some title";
+
+// They can be used to separate multiple nodes
+title "Some title"; author "Peter Parker"
+
+// Tags may contain certain non-alphanumeric characters
+this-is_a.valid$tag-name
+
+// Namespaces are supported
+renderer:options "invisible"
+physics:options "nocollide"
+
+// Nodes can be separated into multiple lines
+title \
+		"Some title"
+

test/results/lf/many_things.sdl

@@ -0,0 +1,64 @@
+name "vibe-d"
+description "Event driven web and concurrency framework"
+homepage "https://vibed.org/"
+
+license "MIT"
+copyright "Copyright © 2012-2020 Sönke Ludwig"
+authors "Sönke Ludwig" "Mathias 'Geod24' Lang" "Etienne Cimon" "Martin Nowak" \
+		"Mihails 'Dicebot' Strasuns" "150 contributors total"
+
+systemDependencies "Optionally OpenSSL 1.1.x"
+dependency ":redis" version="*"
+dependency ":data" version="*"
+dependency ":inet" version="*"
+
+targetType "library"
+targetName "vibed"
+
+// NOTE: "lib" is a path with no D sources to work around an issue in DUB 1.0.0
+//       and below that results in the standard "source/" path to be added even
+//       if an explicit "sourcePaths" directive is given.
+sourcePaths "lib"
+sourceFiles "source/vibe/d.d" "source/vibe/vibe.d"
+
+x:ddoxFilterArgs "--unittest-examples" "--min-protection=Protected" \
+		"--ex" "vibe.core.drivers." "--ex" "vibe.internal." "--ex" "vibe.web.internal." \
+		"--ex" "diet.internal" "--ex" "stdx." "--ex" "eventcore.internal." "--ex" "eventcore.drivers." \
+		"--ex" "mir." "--ex" "openssl_version"
+
+configuration "unittest" {
+	targetPath "bin"
+	targetName "ut"
+
+	dflags "-preview=dip1000"
+
+	sourcePaths "source" "test/ut"
+	importPaths "source" "test"
+	stringImportPaths "test/testFiles"
+
+	foo {
+		bar {
+			""
+		}
+	}
+	versions "unitUnthreaded"
+
+	dependency "unit-threaded" version="*"
+	dependency "unit-threaded:autorunner" version="*"
+}
+// cool beans
+
+/* foo */
+foo {
+	// ok
+	bar {
+		// xd
+		x {
+			d
+		}
+	}
+}
+
+subPackage "utils"
+subPackage "data"
+subPackage "stream"

test/results/tempws-1/syntax_details.sdl

@@ -0,0 +1,17 @@
+// Trailing semicolons are optional
+title "Some title";
+
+// They can be used to separate multiple nodes
+title "Some title"; author "Peter Parker"
+
+// Tags may contain certain non-alphanumeric characters
+this-is_a.valid$tag-name
+
+// Namespaces are supported
+renderer:options "invisible"
+physics:options "nocollide"
+
+// Nodes can be separated into multiple lines
+title \
+	"Some title"
+

test/source/comment_styles.sdl

@@ -0,0 +1,11 @@
+// C++ style
+
+/*
+C style multiline
+*/
+
+tag /*foo=true*/ bar=false
+
+# Shell style
+
+-- Lua style