fix: keep letter-spaced words whole on runs with explicit spaces (#377)

src/parse/page_item_sanitators/cells.h

@@ -896,16 +896,134 @@ namespace pdflib
         // inferred threshold past every word gap in a short heading.
         const double word_gap = std::min(
           0.35, run.has_semantic_spaces ? 1.0 : infer_word_gap(run));
+        // A gap in units of the font space width of the glyphs either side.
+        const auto space_units = [](const page_item<PAGE_CELL>& lhs,
+                                    const page_item<PAGE_CELL>& rhs) -> double
+          {
+            const double unit =
+              0.5 * (cell_space_width(lhs) + cell_space_width(rhs));
+            return std::max(0.0, forward_gap(lhs, rhs)) / std::max(1.e-6, unit);
+          };
+        const auto median_of = [](std::vector<double> values) -> double
+          {
+            std::sort(values.begin(), values.end());
+            const std::size_t n = values.size();
+            return n % 2 ? values[n / 2] : 0.5 * (values[n / 2 - 1] + values[n / 2]);
+          };
+        // On a spaced run: the median effective width of its explicit spaces
+        // (1.0 when none is measurable, e.g. a table cell whose only space
+        // trails it; -1.0 when every space is collapsed by Tw and so says
+        // nothing), and its unmarked gaps, which are wide only when the run is
+        // letter-spaced.
+        double space_reference = -1.0;
+        // Each unmarked gap is kept with its font: a gap between two fonts is no
+        // letter-spacing evidence for either, and one font's tracking must not
+        // vouch for another font's gap. Each is also kept with its segment (the
+        // stretch between explicit spaces), so a gap can be compared with the
+        // letter gaps of its own word.
+        // Grouped and sorted once per run, so the judgement below is a lookup
+        // and not a copy and a sort per candidate boundary. Segment groups are
+        // indexed by segment number, so finding one does not scan the line.
+        using font_groups = std::vector<std::pair<std::string, std::vector<double>>>;
+        font_groups unmarked;
+        std::vector<font_groups> in_segment;
+        if(run.has_semantic_spaces)
+          {
+            std::vector<double> spaces;
+            const page_item<PAGE_CELL>* before = nullptr;
+            bool crossed = false;
+            std::size_t segment = 0;
+            for(const auto& c : run.cells)
+              {
+                if(is_semantic_space(c)) { crossed = true; ++segment; continue; }
+                if(before != nullptr)
+                  {
+                    if(crossed) { spaces.push_back(space_units(*before, c)); }
+                    else if(before->font_name == c.font_name)
+                      {
+                        const double gap = space_units(*before, c);
+                        const auto same_font = [&c](const auto& g) { return g.first == c.font_name; };
+                        auto group = std::find_if(unmarked.begin(), unmarked.end(), same_font);
+                        if(group == unmarked.end())
+                          {
+                            unmarked.emplace_back(c.font_name, std::vector<double>());
+                            group = std::prev(unmarked.end());
+                          }
+                        group->second.push_back(gap);
+                        if(in_segment.size() <= segment) { in_segment.resize(segment + 1); }
+                        font_groups& fonts = in_segment[segment];
+                        auto local = std::find_if(fonts.begin(), fonts.end(), same_font);
+                        if(local == fonts.end())
+                          {
+                            fonts.emplace_back(c.font_name, std::vector<double>());
+                            local = std::prev(fonts.end());
+                          }
+                        local->second.push_back(gap);
+                      }
+                  }
+                before = &c;
+                crossed = false;
+              }
+            for(auto& group : unmarked) { std::sort(group.second.begin(), group.second.end()); }
+            for(auto& fonts : in_segment)
+              {
+                for(auto& group : fonts) { std::sort(group.second.begin(), group.second.end()); }
+              }
+            std::vector<double> usable;
+            for(double w : spaces) { if(w > 0.25) { usable.push_back(w); } }
+            if(spaces.empty()) { space_reference = 1.0; }
+            else if(not usable.empty()) { space_reference = median_of(usable); }
+          }
+        // The number and median of the gaps in a sorted group other than the
+        // gap under test, read off the sorted list with its one position skipped.
+        const auto others = [](const std::vector<double>& sorted,
+                               double gap) -> std::pair<std::size_t, double>
+          {
+            const std::size_t at = static_cast<std::size_t>(
+              std::lower_bound(sorted.begin(), sorted.end(), gap - 1.e-9) - sorted.begin());
+            const bool found = at < sorted.size() and std::abs(sorted[at] - gap) < 1.e-9;
+            const std::size_t n = sorted.size() - (found ? 1 : 0);
+            if(n == 0) { return {0, 0.0}; }
+            const auto other = [&sorted, at, found](std::size_t i) -> double
+              { return sorted[(found and i >= at) ? i + 1 : i]; };
+            return {n, n % 2 ? other(n / 2) : 0.5 * (other(n / 2 - 1) + other(n / 2))};
+          };
+        // A run is letter-spaced only on evidence from its OTHER unmarked
+        // gaps IN THE SAME FONT: at least three of them, the gap under test
+        // left out, so a lone kerned gap cannot vouch for itself and a
+        // differently-fonted tracked span cannot vouch for it either. And the
+        // gap must not stand out from the other letter gaps of its own segment:
+        // tracking widens every gap of a word, so a gap wider than its
+        // neighbours is a word boundary however tracked the rest of the line is.
+        const auto letter_spaced_like = [&unmarked, &in_segment, &others](
+          double gap, const std::string& font, std::size_t segment) -> bool
+          {
+            const auto group = std::find_if(unmarked.begin(), unmarked.end(),
+                                            [&font](const auto& g) { return g.first == font; });
+            if(group == unmarked.end()) { return false; }
+            const auto [n, typical] = others(group->second, gap);
+            if(n < 3 or typical < 0.3 or gap > 1.5 * typical) { return false; }
+            if(segment >= in_segment.size()) { return true; }
+            const font_groups& fonts = in_segment[segment];
+            const auto local = std::find_if(fonts.begin(), fonts.end(),
+                                            [&font](const auto& g) { return g.first == font; });
+            if(local == fonts.end()) { return true; }
+            const auto [n_local, neighbours] = others(local->second, gap);
+            return n_local == 0 or gap <= 1.5 * neighbours;
+          };
+
         std::vector<page_item<PAGE_CELL>> words;
         page_item<PAGE_CELL>* previous_visible = nullptr;
         bool explicit_boundary = false;
+        std::size_t segment = 0;
 
         for(std::size_t index = 0; index < run.cells.size(); ++index)
           {
             auto& cell = run.cells[index];
             if(is_semantic_space(cell))
               {
                 explicit_boundary = true;
+                ++segment;
                 continue;
               }
 
@@ -938,6 +1056,28 @@ namespace pdflib
                       std::max(0.0, forward_gap(*previous_visible, cell)) /
                       std::max(1.e-6, scale);
                     starts_word = normalized_gap > word_gap;
+                    // On a spaced run, drop a boundary the gap does not earn:
+                    // either under half a font space and half the run's own
+                    // space (the reference is capped at 1.3 because justified
+                    // stretch and table gutters inflate the median), or under
+                    // 0.8 of the run's own space and no wider than a
+                    // letter-spaced run's usual gap (judged from the run's
+                    // other gaps and its segment's). Real tracking is often
+                    // 0.6-0.8 of a space, so half a space is too tight for the
+                    // second; at a full space positioned maths symbols join.
+                    // Never across a font change.
+                    // This can only remove a split.
+                    if(starts_word and space_reference > 0.0 and
+                       previous_visible->font_name == cell.font_name)
+                      {
+                        const double gap = space_units(*previous_visible, cell);
+                        const bool narrow =
+                          gap < 0.5 * std::min(space_reference, 1.3);
+                        const bool lettered =
+                          gap < 0.8 * space_reference and
+                          letter_spaced_like(gap, cell.font_name, segment);
+                        starts_word = not (narrow or lettered);
+                      }
                   }
               }
 

tests/test_unit_word_gap_letter_spacing.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+"""Word boundaries on a spaced run whose glyphs are letter-spaced (docling-parse#377).
+
+On a run that carries explicit spaces, a gap between two glyphs used to start a
+new word whenever it passed the word-gap threshold, so a tracked word such as
+`T R A C K I N G` came out as one word per letter. The contraction now drops
+such a boundary when the gap is small against the run's own explicit spaces and
+is either under half a font space or no wider than the font's usual letter gap
+on that run. Two bounds keep the second test from merging real words:
+
+* the evidence comes only from other gaps in the same font, so a tracked span
+  in another font cannot vouch for a gap;
+* the gap must not stand out from the other letter gaps of its own segment
+  (the glyphs between two explicit spaces), so a tracked span elsewhere on the
+  line cannot vouch for a gap between two untracked words.
+
+Every page below is built in memory with core-14 fonts, so each test reads as
+the content stream it is about. The streams come from the review of #377. /F1
+is Helvetica and /F2 is Times-Roman; at 12 pt a TJ adjustment of -278 moves the
+next glyph by one Helvetica space (278/1000 em), and `14 Tw` widens the
+explicit space after `X` so the run's space reference is several spaces wide.
+"""
+
+from tests.pdf_builder import parse_page, simple_page_pdf
+
+HELVETICA = (
+    "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"
+)
+TIMES = "<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>"
+
+TRACKED = "[(T) -278 (R) -278 (A) -278 (C) -278 (K) -278 (I) -278 (N) -278 (G)] TJ"
+ONE_TWO = "14 Tw (X ) Tj 0 Tw [(ONE) -278 (TWO)] TJ ( ) Tj"
+SPLIT_ONE_TWO = ["X", "ONE", "TWO", "TRACKING"]
+
+
+def _words(content: str) -> list[str]:
+    pdf = simple_page_pdf(
+        f"BT /F1 12 Tf 1 0 0 1 50 50 Tm {content} ET",
+        resources="/Font << /F1 5 0 R /F2 6 0 R >>",
+        media_box="[0 0 300 150]",
+        extra_objects=[HELVETICA, TIMES],
+    )
+    return [cell.text for cell in parse_page(pdf).word_cells]
+
+
+def test_tracked_word_in_the_same_font_rejoins():
+    # The repair itself: every letter gap of TRACKING is one space, as wide as
+    # the gap between ONE and TWO, but only TRACKING has letter gaps that wide.
+    assert _words(f"{ONE_TWO} /F1 12 Tf {TRACKED}") == SPLIT_ONE_TWO
+
+
+def test_plain_suffix_leaves_one_two_split():
+    assert _words(f"{ONE_TWO} /F1 12 Tf (TRACKING) Tj") == SPLIT_ONE_TWO
+
+
+def test_tracked_word_in_another_font_does_not_vouch():
+    assert _words(f"{ONE_TWO} /F2 12 Tf {TRACKED}") == SPLIT_ONE_TWO
+
+
+def test_plain_suffix_in_another_font():
+    assert _words(f"{ONE_TWO} /F2 12 Tf (TRACKING) Tj") == SPLIT_ONE_TWO
+
+
+def test_single_kerned_gap_after_expanded_space_stays_split():
+    # One gap of a nominal space and nothing else to judge it by: a lone kerned
+    # gap cannot vouch for itself.
+    assert _words("14 Tw (X ) Tj 0 Tw [(A) -278 (B)] TJ") == ["X", "A", "B"]
+
+
+def test_small_text_after_large_spaced_text_stays_split():
+    # The spaces of 48 pt text say nothing about a gap in 12 pt text.
+    pdf = simple_page_pdf(
+        "BT /F1 48 Tf 1 0 0 1 50 50 Tm (X Y ) Tj /F1 12 Tf [(A) -278 (B)] TJ ET",
+        resources="/Font << /F1 5 0 R >>",
+        media_box="[0 0 300 150]",
+        extra_objects=[HELVETICA],
+    )
+    assert [cell.text for cell in parse_page(pdf).word_cells] == ["X", "Y", "A", "B"]
+
+
+def test_space_collapsed_by_negative_word_spacing_is_no_reference():
+    # -3.336 Tw cancels the space's advance, so the run has no usable space
+    # width and the rule must not act; the kerned HE / LLO stays as before.
+    assert _words("[(HE) -166.6666667 (LLO)] TJ -3.336 Tw ( X) Tj") == ["HELLO", "X"]
+
+
+def test_tracking_under_a_measured_space_rejoins():
+    # Ordinary explicit spaces, so the run's space reference is one font space,
+    # and the letter gaps of `little` are 0.7 of it (-195 = 0.7 x 278): wider
+    # than half a space, which real tracking routinely is, and still one word.
+    # Narrow letters, so each gap also passes the word-gap threshold.
+    tracked = "[(l) -195 (i) -195 (t) -195 (t) -195 (l) -195 (e)] TJ"
+    assert _words(f"(AB CD ) Tj {tracked}") == ["AB", "CD", "little"]
+
+
+def test_untracked_words_at_the_same_gap_stay_split():
+    # The same 0.7-space gap between two untracked words: no other gap on the
+    # run is that wide, so nothing vouches for it.
+    assert _words("(AB CD ) Tj [(lit) -195 (tilt)] TJ") == ["AB", "CD", "lit", "tilt"]
+
+
+def test_uniform_tracking_across_a_boundary_is_not_detectable():
+    # The documented limit: when ONE and TWO are tracked by the same amount as
+    # the gap between them, nothing on the page marks that boundary, and the
+    # rule joins them. Kept here so that a change to it is a visible decision.
+    content = (
+        "14 Tw (X ) Tj 0 Tw "
+        "[(O) -278 (N) -278 (E) -278 (T) -278 (W) -278 (O)] TJ ( ) Tj "
+        f"{TRACKED}"
+    )
+    assert _words(content) == ["X", "ONETWO", "TRACKING"]