pkg/md: Add more tests against reflowing.

Also fix a bug where the reflow can result in lines one unit wider than the
budget.

Author: xiaq

pkg/md/fmt.go

@@ -662,15 +662,39 @@ func (c *FmtCodec) writeSegmentsParagraphReflow(segs []segment, maxWidth int) {
 		if currentLine.Len() == 0 {
 			currentLine.WriteString(span)
 			currentLineWidth = w
-		} else if currentLineWidth+1+w <= maxWidth {
-			currentLine.WriteByte(' ')
-			currentLine.WriteString(span)
-			currentLineWidth += 1 + w
 		} else {
-			writeCurrentLine()
-			startNewLine()
-			currentLine.WriteString(span)
-			currentLineWidth = w
+			// Determine whether the current span fits onto the current line.
+			//
+			// One slightly tricky detail here is that c.escapeStartOfLine may
+			// insert more text, making the line wider. In reflow mode, the line
+			// never starts or ends with whitespaces, so the most we have to
+			// worry about is one backslash.
+			//
+			// As a result, if the line's width is exactly maxWidth after
+			// appending the current span, we need to be extra careful and only
+			// consider the current span to fit if c.escapeStartOfLine won't
+			// introduce an additional backslash.
+			//
+			// The current implementation of this check is rather inefficient,
+			// but since the check is done at most once per line, the
+			// performance might as well be good enough.
+			fits := false
+			if currentLineWidth+1+w < maxWidth {
+				fits = true
+			} else if currentLineWidth+1+w == maxWidth {
+				line := currentLine.String() + " " + span
+				fits = c.escapeStartOfLine(line, startOfParagraph, true) == line
+			}
+			if fits {
+				currentLine.WriteByte(' ')
+				currentLine.WriteString(span)
+				currentLineWidth += 1 + w
+			} else {
+				writeCurrentLine()
+				startNewLine()
+				currentLine.WriteString(span)
+				currentLineWidth = w
+			}
 		}
 		if hardLineBreak {
 			writeCurrentLine()

pkg/md/fmt_test.go

@@ -1,6 +1,7 @@
 package md_test
 
 import (
+	"fmt"
 	"html"
 	"regexp"
 	"strings"
@@ -10,6 +11,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	. "src.elv.sh/pkg/md"
 	"src.elv.sh/pkg/testutil"
+	"src.elv.sh/pkg/wcwidth"
 )
 
 var supplementalFmtCases = []testCase{
@@ -91,43 +93,38 @@ func TestFmtPreservesHTMLRender(t *testing.T) {
 	}
 }
 
-func TestReflowFmtPreservesHTMLRenderModuleWhitespaces(t *testing.T) {
-	testutil.Set(t, &UnescapeHTML, html.UnescapeString)
-	for _, tc := range fmtTestCases {
-		t.Run(tc.testName(), func(t *testing.T) {
-			testReflowFmtPreservesHTMLRenderModuloWhitespaces(t, tc.Markdown, 80)
-		})
-	}
-}
-
 func FuzzFmtPreservesHTMLRender(f *testing.F) {
 	for _, tc := range fmtTestCases {
 		f.Add(tc.Markdown)
 	}
 	f.Fuzz(testFmtPreservesHTMLRender)
 }
 
-func FuzzReflowFmtPreservesHTMLRenderModuleWhitespaces(f *testing.F) {
-	for _, tc := range fmtTestCases {
-		f.Add(tc.Markdown, 20)
-		f.Add(tc.Markdown, 80)
-	}
-	f.Fuzz(testReflowFmtPreservesHTMLRenderModuloWhitespaces)
-}
-
 func testFmtPreservesHTMLRender(t *testing.T, original string) {
-	t.Helper()
 	testFmtPreservesHTMLRenderModulo(t, original, 0, nil)
 }
 
+func TestReflowFmtPreservesHTMLRenderModuleWhitespaces(t *testing.T) {
+	testReflowFmt(t, testReflowFmtPreservesHTMLRenderModuloWhitespaces)
+}
+
+func FuzzReflowFmtPreservesHTMLRenderModuleWhitespaces(f *testing.F) {
+	fuzzReflowFmt(f, testReflowFmtPreservesHTMLRenderModuloWhitespaces)
+}
+
 var (
 	paragraph         = regexp.MustCompile(`(?s)<p>.*?</p>`)
 	whitespaceRun     = regexp.MustCompile(`[ \t\n]+`)
 	brWithWhitespaces = regexp.MustCompile(`[ \t\n]*<br />[ \t\n]*`)
 )
 
 func testReflowFmtPreservesHTMLRenderModuloWhitespaces(t *testing.T, original string, w int) {
-	t.Helper()
+	if strings.Contains(original, "<p>") {
+		t.Skip("markdown contains <p>")
+	}
+	if strings.Contains(original, "</p>") {
+		t.Skip("markdown contains </p>")
+	}
 	testFmtPreservesHTMLRenderModulo(t, original, w, func(html string) string {
 		// Coalesce whitespaces in each paragraph.
 		return paragraph.ReplaceAllStringFunc(html, func(p string) string {
@@ -141,8 +138,98 @@ func testReflowFmtPreservesHTMLRenderModuloWhitespaces(t *testing.T, original st
 	})
 }
 
+func TestReflowFmtResultIsUnchangedUnderFmt(t *testing.T) {
+	testReflowFmt(t, testReflowFmtResultIsUnchangedUnderFmt)
+}
+
+func FuzzReflowFmtResultIsUnchangedUnderFmt(f *testing.F) {
+	fuzzReflowFmt(f, testReflowFmtResultIsUnchangedUnderFmt)
+}
+
+func testReflowFmtResultIsUnchangedUnderFmt(t *testing.T, original string, w int) {
+	reflowed := formatAndSkipIfUnsupported(t, original, w)
+	formatted := RenderString(reflowed, &FmtCodec{})
+	if reflowed != formatted {
+		t.Errorf("original:\n%s\nreflowed:\n%s\nformatted:\n%s"+
+			"markdown diff (-reflowed +formatted):\n%s",
+			hr+"\n"+original+hr, hr+"\n"+reflowed+hr, hr+"\n"+formatted+hr,
+			cmp.Diff(reflowed, formatted))
+	}
+}
+
+func TestReflowFmtResultFitsInWidth(t *testing.T) {
+	testReflowFmt(t, testReflowFmtResultFitsInWidth)
+}
+
+func FuzzReflowFmtResultFitsInWidth(f *testing.F) {
+	fuzzReflowFmt(f, testReflowFmtResultFitsInWidth)
+}
+
+var (
+	// Match all markers that can be written by FmtCodec.
+	markersRegexp  = regexp.MustCompile(`^ *(?:(?:[-*>]|[0-9]{1,9}[.)]) *)*`)
+	linkRegexp     = regexp.MustCompile(`\[.*\]\(.*\)`)
+	codeSpanRegexp = regexp.MustCompile("`.*`")
+)
+
+func testReflowFmtResultFitsInWidth(t *testing.T, original string, w int) {
+	if w <= 0 {
+		t.Skip("width <= 0")
+	}
+
+	var trace TraceCodec
+	Render(original, &trace)
+	for _, op := range trace.Ops() {
+		switch op.Type {
+		case OpHeading, OpCodeBlock, OpHTMLBlock:
+			t.Skipf("input contains unsupported block type %s", op.Type)
+		}
+	}
+
+	reflowed := formatAndSkipIfUnsupported(t, original, w)
+
+	for _, line := range strings.Split(reflowed, "\n") {
+		lineWidth := wcwidth.Of(line)
+		if lineWidth <= w {
+			continue
+		}
+		// Strip all markers
+		content := line[len(markersRegexp.FindString(line)):]
+		// Analyze whether the content is allowed to exceed width
+		switch {
+		case !strings.Contains(content, " "):
+		case strings.Contains(content, "<"):
+		case linkRegexp.MatchString(content):
+		case codeSpanRegexp.MatchString(content):
+		default:
+			t.Errorf("line length > %d: %q\nfull reflowed:\n%s",
+				w, line, hr+"\n"+reflowed+hr)
+		}
+	}
+}
+
+var widths = []int{20, 51, 80}
+
+func testReflowFmt(t *testing.T, test func(*testing.T, string, int)) {
+	for _, tc := range fmtTestCases {
+		for _, w := range widths {
+			t.Run(fmt.Sprintf("%s/Width %d", tc.testName(), w), func(t *testing.T) {
+				test(t, tc.Markdown, w)
+			})
+		}
+	}
+}
+
+func fuzzReflowFmt(f *testing.F, test func(*testing.T, string, int)) {
+	for _, tc := range fmtTestCases {
+		for _, w := range widths {
+			f.Add(tc.Markdown, w)
+		}
+	}
+	f.Fuzz(test)
+}
+
 func testFmtPreservesHTMLRenderModulo(t *testing.T, original string, w int, processHTML func(string) string) {
-	t.Helper()
 	formatted := formatAndSkipIfUnsupported(t, original, w)
 	originalRender := RenderString(original, &HTMLCodec{})
 	formattedRender := RenderString(formatted, &HTMLCodec{})
@@ -163,7 +250,6 @@ func testFmtPreservesHTMLRenderModulo(t *testing.T, original string, w int, proc
 }
 
 func formatAndSkipIfUnsupported(t *testing.T, original string, w int) string {
-	t.Helper()
 	if !utf8.ValidString(original) {
 		t.Skipf("input is not valid UTF-8")
 	}

pkg/md/testdata/fuzz/FuzzReflowFmtPreservesHTMLRenderModuleWhitespaces/d0138fd295a0fa831075f449614b0a8b8cf421e8754463f1907d68ef05ef0475

@@ -0,0 +1,3 @@
+go test fuzz v1
+string("0</p>\n0")
+int(80)

pkg/md/testdata/fuzz/FuzzReflowFmtResultFitsInWidth/4addab28ca79b43124a432b0980b1504d78dc5ea7b8c722d4a3f960729240178

@@ -0,0 +1,3 @@
+go test fuzz v1
+string("\x02 00")
+int(-118)

pkg/md/testdata/fuzz/FuzzReflowFmtResultFitsInWidth/c5523be82830755cd690a04810119e39290f0dc446c38e612dc0cc46ccc4cdaf

@@ -0,0 +1,3 @@
+go test fuzz v1
+string("\\# 000")
+int(5)

pkg/md/trace.go

@@ -6,9 +6,13 @@ import (
 )
 
 // TraceCodec is a Codec that records all the Op's passed to its Do method.
-type TraceCodec struct{ strings.Builder }
+type TraceCodec struct {
+	ops []Op
+	strings.Builder
+}
 
 func (c *TraceCodec) Do(op Op) {
+	c.ops = append(c.ops, op)
 	c.WriteString(op.Type.String())
 	if op.Number != 0 {
 		fmt.Fprintf(c, " Number=%d", op.Number)
@@ -37,3 +41,5 @@ func (c *TraceCodec) Do(op Op) {
 		c.WriteString("\n")
 	}
 }
+
+func (c *TraceCodec) Ops() []Op { return c.ops }