diff --git a/src/parse/page_item_sanitators/cells.h b/src/parse/page_item_sanitators/cells.h index 355f02e9..884e674c 100644 --- a/src/parse/page_item_sanitators/cells.h +++ b/src/parse/page_item_sanitators/cells.h @@ -896,9 +896,126 @@ namespace pdflib // inferred threshold past every word gap in a short heading. const double word_gap = std::min( 0.35, run.has_semantic_spaces ? 1.0 : infer_word_gap(run)); + // A gap in units of the font space width of the glyphs either side. + const auto space_units = [](const page_item& lhs, + const page_item& rhs) -> double + { + const double unit = + 0.5 * (cell_space_width(lhs) + cell_space_width(rhs)); + return std::max(0.0, forward_gap(lhs, rhs)) / std::max(1.e-6, unit); + }; + const auto median_of = [](std::vector values) -> double + { + std::sort(values.begin(), values.end()); + const std::size_t n = values.size(); + return n % 2 ? values[n / 2] : 0.5 * (values[n / 2 - 1] + values[n / 2]); + }; + // On a spaced run: the median effective width of its explicit spaces + // (1.0 when none is measurable, e.g. a table cell whose only space + // trails it; -1.0 when every space is collapsed by Tw and so says + // nothing), and its unmarked gaps, which are wide only when the run is + // letter-spaced. + double space_reference = -1.0; + // Each unmarked gap is kept with its font: a gap between two fonts is no + // letter-spacing evidence for either, and one font's tracking must not + // vouch for another font's gap. Each is also kept with its segment (the + // stretch between explicit spaces), so a gap can be compared with the + // letter gaps of its own word. + // Grouped and sorted once per run, so the judgement below is a lookup + // and not a copy and a sort per candidate boundary. Segment groups are + // indexed by segment number, so finding one does not scan the line. + using font_groups = std::vector>>; + font_groups unmarked; + std::vector in_segment; + if(run.has_semantic_spaces) + { + std::vector spaces; + const page_item* before = nullptr; + bool crossed = false; + std::size_t segment = 0; + for(const auto& c : run.cells) + { + if(is_semantic_space(c)) { crossed = true; ++segment; continue; } + if(before != nullptr) + { + if(crossed) { spaces.push_back(space_units(*before, c)); } + else if(before->font_name == c.font_name) + { + const double gap = space_units(*before, c); + const auto same_font = [&c](const auto& g) { return g.first == c.font_name; }; + auto group = std::find_if(unmarked.begin(), unmarked.end(), same_font); + if(group == unmarked.end()) + { + unmarked.emplace_back(c.font_name, std::vector()); + group = std::prev(unmarked.end()); + } + group->second.push_back(gap); + if(in_segment.size() <= segment) { in_segment.resize(segment + 1); } + font_groups& fonts = in_segment[segment]; + auto local = std::find_if(fonts.begin(), fonts.end(), same_font); + if(local == fonts.end()) + { + fonts.emplace_back(c.font_name, std::vector()); + local = std::prev(fonts.end()); + } + local->second.push_back(gap); + } + } + before = &c; + crossed = false; + } + for(auto& group : unmarked) { std::sort(group.second.begin(), group.second.end()); } + for(auto& fonts : in_segment) + { + for(auto& group : fonts) { std::sort(group.second.begin(), group.second.end()); } + } + std::vector usable; + for(double w : spaces) { if(w > 0.25) { usable.push_back(w); } } + if(spaces.empty()) { space_reference = 1.0; } + else if(not usable.empty()) { space_reference = median_of(usable); } + } + // The number and median of the gaps in a sorted group other than the + // gap under test, read off the sorted list with its one position skipped. + const auto others = [](const std::vector& sorted, + double gap) -> std::pair + { + const std::size_t at = static_cast( + std::lower_bound(sorted.begin(), sorted.end(), gap - 1.e-9) - sorted.begin()); + const bool found = at < sorted.size() and std::abs(sorted[at] - gap) < 1.e-9; + const std::size_t n = sorted.size() - (found ? 1 : 0); + if(n == 0) { return {0, 0.0}; } + const auto other = [&sorted, at, found](std::size_t i) -> double + { return sorted[(found and i >= at) ? i + 1 : i]; }; + return {n, n % 2 ? other(n / 2) : 0.5 * (other(n / 2 - 1) + other(n / 2))}; + }; + // A run is letter-spaced only on evidence from its OTHER unmarked + // gaps IN THE SAME FONT: at least three of them, the gap under test + // left out, so a lone kerned gap cannot vouch for itself and a + // differently-fonted tracked span cannot vouch for it either. And the + // gap must not stand out from the other letter gaps of its own segment: + // tracking widens every gap of a word, so a gap wider than its + // neighbours is a word boundary however tracked the rest of the line is. + const auto letter_spaced_like = [&unmarked, &in_segment, &others]( + double gap, const std::string& font, std::size_t segment) -> bool + { + const auto group = std::find_if(unmarked.begin(), unmarked.end(), + [&font](const auto& g) { return g.first == font; }); + if(group == unmarked.end()) { return false; } + const auto [n, typical] = others(group->second, gap); + if(n < 3 or typical < 0.3 or gap > 1.5 * typical) { return false; } + if(segment >= in_segment.size()) { return true; } + const font_groups& fonts = in_segment[segment]; + const auto local = std::find_if(fonts.begin(), fonts.end(), + [&font](const auto& g) { return g.first == font; }); + if(local == fonts.end()) { return true; } + const auto [n_local, neighbours] = others(local->second, gap); + return n_local == 0 or gap <= 1.5 * neighbours; + }; + std::vector> words; page_item* previous_visible = nullptr; bool explicit_boundary = false; + std::size_t segment = 0; for(std::size_t index = 0; index < run.cells.size(); ++index) { @@ -906,6 +1023,7 @@ namespace pdflib if(is_semantic_space(cell)) { explicit_boundary = true; + ++segment; continue; } @@ -938,6 +1056,28 @@ namespace pdflib std::max(0.0, forward_gap(*previous_visible, cell)) / std::max(1.e-6, scale); starts_word = normalized_gap > word_gap; + // On a spaced run, drop a boundary the gap does not earn: + // either under half a font space and half the run's own + // space (the reference is capped at 1.3 because justified + // stretch and table gutters inflate the median), or under + // 0.8 of the run's own space and no wider than a + // letter-spaced run's usual gap (judged from the run's + // other gaps and its segment's). Real tracking is often + // 0.6-0.8 of a space, so half a space is too tight for the + // second; at a full space positioned maths symbols join. + // Never across a font change. + // This can only remove a split. + if(starts_word and space_reference > 0.0 and + previous_visible->font_name == cell.font_name) + { + const double gap = space_units(*previous_visible, cell); + const bool narrow = + gap < 0.5 * std::min(space_reference, 1.3); + const bool lettered = + gap < 0.8 * space_reference and + letter_spaced_like(gap, cell.font_name, segment); + starts_word = not (narrow or lettered); + } } } diff --git a/tests/test_unit_word_gap_letter_spacing.py b/tests/test_unit_word_gap_letter_spacing.py new file mode 100644 index 00000000..add18d92 --- /dev/null +++ b/tests/test_unit_word_gap_letter_spacing.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +"""Word boundaries on a spaced run whose glyphs are letter-spaced (docling-parse#377). + +On a run that carries explicit spaces, a gap between two glyphs used to start a +new word whenever it passed the word-gap threshold, so a tracked word such as +`T R A C K I N G` came out as one word per letter. The contraction now drops +such a boundary when the gap is small against the run's own explicit spaces and +is either under half a font space or no wider than the font's usual letter gap +on that run. Two bounds keep the second test from merging real words: + +* the evidence comes only from other gaps in the same font, so a tracked span + in another font cannot vouch for a gap; +* the gap must not stand out from the other letter gaps of its own segment + (the glyphs between two explicit spaces), so a tracked span elsewhere on the + line cannot vouch for a gap between two untracked words. + +Every page below is built in memory with core-14 fonts, so each test reads as +the content stream it is about. The streams come from the review of #377. /F1 +is Helvetica and /F2 is Times-Roman; at 12 pt a TJ adjustment of -278 moves the +next glyph by one Helvetica space (278/1000 em), and `14 Tw` widens the +explicit space after `X` so the run's space reference is several spaces wide. +""" + +from tests.pdf_builder import parse_page, simple_page_pdf + +HELVETICA = ( + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>" +) +TIMES = "<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>" + +TRACKED = "[(T) -278 (R) -278 (A) -278 (C) -278 (K) -278 (I) -278 (N) -278 (G)] TJ" +ONE_TWO = "14 Tw (X ) Tj 0 Tw [(ONE) -278 (TWO)] TJ ( ) Tj" +SPLIT_ONE_TWO = ["X", "ONE", "TWO", "TRACKING"] + + +def _words(content: str) -> list[str]: + pdf = simple_page_pdf( + f"BT /F1 12 Tf 1 0 0 1 50 50 Tm {content} ET", + resources="/Font << /F1 5 0 R /F2 6 0 R >>", + media_box="[0 0 300 150]", + extra_objects=[HELVETICA, TIMES], + ) + return [cell.text for cell in parse_page(pdf).word_cells] + + +def test_tracked_word_in_the_same_font_rejoins(): + # The repair itself: every letter gap of TRACKING is one space, as wide as + # the gap between ONE and TWO, but only TRACKING has letter gaps that wide. + assert _words(f"{ONE_TWO} /F1 12 Tf {TRACKED}") == SPLIT_ONE_TWO + + +def test_plain_suffix_leaves_one_two_split(): + assert _words(f"{ONE_TWO} /F1 12 Tf (TRACKING) Tj") == SPLIT_ONE_TWO + + +def test_tracked_word_in_another_font_does_not_vouch(): + assert _words(f"{ONE_TWO} /F2 12 Tf {TRACKED}") == SPLIT_ONE_TWO + + +def test_plain_suffix_in_another_font(): + assert _words(f"{ONE_TWO} /F2 12 Tf (TRACKING) Tj") == SPLIT_ONE_TWO + + +def test_single_kerned_gap_after_expanded_space_stays_split(): + # One gap of a nominal space and nothing else to judge it by: a lone kerned + # gap cannot vouch for itself. + assert _words("14 Tw (X ) Tj 0 Tw [(A) -278 (B)] TJ") == ["X", "A", "B"] + + +def test_small_text_after_large_spaced_text_stays_split(): + # The spaces of 48 pt text say nothing about a gap in 12 pt text. + pdf = simple_page_pdf( + "BT /F1 48 Tf 1 0 0 1 50 50 Tm (X Y ) Tj /F1 12 Tf [(A) -278 (B)] TJ ET", + resources="/Font << /F1 5 0 R >>", + media_box="[0 0 300 150]", + extra_objects=[HELVETICA], + ) + assert [cell.text for cell in parse_page(pdf).word_cells] == ["X", "Y", "A", "B"] + + +def test_space_collapsed_by_negative_word_spacing_is_no_reference(): + # -3.336 Tw cancels the space's advance, so the run has no usable space + # width and the rule must not act; the kerned HE / LLO stays as before. + assert _words("[(HE) -166.6666667 (LLO)] TJ -3.336 Tw ( X) Tj") == ["HELLO", "X"] + + +def test_tracking_under_a_measured_space_rejoins(): + # Ordinary explicit spaces, so the run's space reference is one font space, + # and the letter gaps of `little` are 0.7 of it (-195 = 0.7 x 278): wider + # than half a space, which real tracking routinely is, and still one word. + # Narrow letters, so each gap also passes the word-gap threshold. + tracked = "[(l) -195 (i) -195 (t) -195 (t) -195 (l) -195 (e)] TJ" + assert _words(f"(AB CD ) Tj {tracked}") == ["AB", "CD", "little"] + + +def test_untracked_words_at_the_same_gap_stay_split(): + # The same 0.7-space gap between two untracked words: no other gap on the + # run is that wide, so nothing vouches for it. + assert _words("(AB CD ) Tj [(lit) -195 (tilt)] TJ") == ["AB", "CD", "lit", "tilt"] + + +def test_uniform_tracking_across_a_boundary_is_not_detectable(): + # The documented limit: when ONE and TWO are tracked by the same amount as + # the gap between them, nothing on the page marks that boundary, and the + # rule joins them. Kept here so that a change to it is a visible decision. + content = ( + "14 Tw (X ) Tj 0 Tw " + "[(O) -278 (N) -278 (E) -278 (T) -278 (W) -278 (O)] TJ ( ) Tj " + f"{TRACKED}" + ) + assert _words(content) == ["X", "ONETWO", "TRACKING"]