Highlight unusual Unicode whitespace

crates/edit/src/bin/edit/draw_menubar.rs

@@ -140,6 +140,16 @@ fn draw_menu_view(ctx: &mut Context, state: &mut State) {
             tb.set_word_wrap(!word_wrap);
             ctx.needs_rerender();
         }
+        let highlight_unusual_whitespace = tb.is_unusual_whitespace_highlight_enabled();
+        if ctx.menubar_menu_checkbox(
+            loc(LocId::ViewHighlightUnusualWhitespace),
+            'I',
+            kbmod::ALT | vk::I,
+            highlight_unusual_whitespace,
+        ) {
+            tb.set_unusual_whitespace_highlight_enabled(!highlight_unusual_whitespace);
+            ctx.needs_rerender();
+        }
     }
 
     ctx.menubar_menu_end();

crates/edit/src/buffer/mod.rs

@@ -258,6 +258,7 @@ pub struct TextBuffer {
     margin_enabled: bool,
     word_wrap_column: CoordType,
     word_wrap_enabled: bool,
+    unusual_whitespace_highlight_enabled: bool,
     tab_size: CoordType,
     indent_with_tabs: bool,
     line_highlight_enabled: bool,
@@ -308,6 +309,7 @@ impl TextBuffer {
             margin_enabled: false,
             word_wrap_column: 0,
             word_wrap_enabled: false,
+            unusual_whitespace_highlight_enabled: false,
             tab_size: 4,
             indent_with_tabs: false,
             line_highlight_enabled: false,
@@ -538,6 +540,19 @@ impl TextBuffer {
         }
     }
 
+    /// Is highlighting of unusual Unicode whitespace enabled?
+    pub fn is_unusual_whitespace_highlight_enabled(&self) -> bool {
+        self.unusual_whitespace_highlight_enabled
+    }
+
+    /// Enable or disable highlighting of unusual Unicode whitespace.
+    ///
+    /// This only affects how characters are visualized (a stand-in glyph plus a
+    /// yellow background), not the layout, so no reflow is necessary.
+    pub fn set_unusual_whitespace_highlight_enabled(&mut self, enabled: bool) {
+        self.unusual_whitespace_highlight_enabled = enabled;
+    }
+
     /// Set the width available for layout.
     ///
     /// Ideally this would be a pure UI concern, but the text buffer needs this
@@ -1790,6 +1805,7 @@ impl TextBuffer {
         let height = destination.height();
         let line_number_width = self.margin_width.max(3) as usize - 3;
         let text_width = width - self.margin_width;
+        let highlight_unusual_whitespace = self.unusual_whitespace_highlight_enabled;
         let mut visual_pos_x_max = 0;
 
         // Pick the cursor closer to the `origin.y`.
@@ -1869,6 +1885,7 @@ impl TextBuffer {
             }
 
             let mut selection_off = 0..0;
+            let mut selection_highlight = None;
 
             // Figure out the selection range on this line, if any.
             if cursor_beg.visual_pos.y == visual_line
@@ -1919,8 +1936,12 @@ impl TextBuffer {
                     bg = bg.oklab_blend(fb.indexed_alpha(IndexedColor::Background, 1, 2));
                 };
                 let fg = fb.contrasted(bg);
-                fb.blend_bg(rect, bg);
-                fb.blend_fg(rect, fg);
+                if highlight_unusual_whitespace {
+                    selection_highlight = Some((rect, bg, fg));
+                } else {
+                    fb.blend_bg(rect, bg);
+                    fb.blend_fg(rect, fg);
+                }
             }
 
             // Nothing to do if the entire line is empty.
@@ -1965,7 +1986,7 @@ impl TextBuffer {
                             let mut whitespace = TAB_WHITESPACE;
                             let mut prefix_add = 0;
 
-                            if is_tab || visualize {
+                            if is_tab || (visualize && !highlight_unusual_whitespace) {
                                 // We need the character's visual position in order to either compute the tab size,
                                 // or set the foreground color of the visualizer, respectively.
                                 // TODO: Doing this char-by-char is bad for performance.
@@ -1985,19 +2006,22 @@ impl TextBuffer {
                                     (VISUAL_SPACE, VISUAL_SPACE_PREFIX_ADD)
                                 };
 
-                                // Make the visualized characters slightly gray.
-                                let visualizer_rect = {
-                                    let left = destination.left
-                                        + self.margin_width
-                                        + cursor_line.visual_pos.x
-                                        - origin.x;
-                                    let top = destination.top + cursor_line.visual_pos.y - origin.y;
-                                    Rect { left, top, right: left + 1, bottom: top + 1 }
-                                };
-                                fb.blend_fg(
-                                    visualizer_rect,
-                                    fb.indexed_alpha(IndexedColor::Foreground, 1, 2),
-                                );
+                                if !highlight_unusual_whitespace {
+                                    // Make the visualized characters slightly gray.
+                                    let visualizer_rect = {
+                                        let left = destination.left
+                                            + self.margin_width
+                                            + cursor_line.visual_pos.x
+                                            - origin.x;
+                                        let top =
+                                            destination.top + cursor_line.visual_pos.y - origin.y;
+                                        Rect { left, top, right: left + 1, bottom: top + 1 }
+                                    };
+                                    fb.blend_fg(
+                                        visualizer_rect,
+                                        fb.indexed_alpha(IndexedColor::Foreground, 1, 2),
+                                    );
+                                }
                             }
 
                             line.extend_from_slice(
@@ -2014,7 +2038,23 @@ impl TextBuffer {
                 visual_pos_x_max = visual_pos_x_max.max(cursor_end.visual_pos.x);
             }
 
-            fb.replace_text(destination.top + y, destination.left, destination.right, &line);
+            if highlight_unusual_whitespace {
+                fb.replace_text_with_unusual_whitespace_highlight(
+                    destination.top + y,
+                    destination.left,
+                    destination.right,
+                    &line,
+                );
+            } else {
+                fb.replace_text(destination.top + y, destination.left, destination.right, &line);
+            }
+
+            // Selection takes precedence over sanitizer highlights, so unusual
+            // whitespace remains visibly selected instead of staying solid yellow.
+            if let Some((rect, bg, fg)) = selection_highlight {
+                fb.blend_bg(rect, bg);
+                fb.blend_fg(rect, fg);
+            }
 
             cursor = cursor_end;
         }

crates/edit/src/framebuffer.rs

@@ -11,7 +11,9 @@ use std::slice::ChunksExact;
 use stdext::arena::{Arena, scratch_arena};
 use stdext::collections::BString;
 use stdext::simd::memset;
-use stdext::unicode::{SanitizedControlChars, sanitize_control_chars};
+use stdext::unicode::{
+    SanitizedControlChars, sanitize_control_chars, sanitize_control_chars_with_unusual_whitespace,
+};
 use stdext::{MaybeOwned, arena_write_fmt};
 
 use crate::helpers::{CoordType, Point, Rect, Size};
@@ -195,7 +197,7 @@ impl Framebuffer {
 
     /// Replaces text contents in a single line of the framebuffer.
     /// All coordinates are in viewport coordinates.
-    /// Assumes that control characters have been replaced or escaped.
+    /// Control characters and invalid UTF-8 are visualized before rendering.
     #[inline]
     pub fn replace_text(
         &mut self,
@@ -204,7 +206,20 @@ impl Framebuffer {
         clip_right: CoordType,
         text: &(impl AsRef<[u8]> + ?Sized),
     ) {
-        self.replace_text_impl(y, origin_x, clip_right, text.as_ref());
+        self.replace_text_impl(y, origin_x, clip_right, text.as_ref(), false);
+    }
+
+    /// Replaces text and highlights unusual Unicode whitespace in a single line.
+    /// All coordinates are in viewport coordinates.
+    #[inline]
+    pub fn replace_text_with_unusual_whitespace_highlight(
+        &mut self,
+        y: CoordType,
+        origin_x: CoordType,
+        clip_right: CoordType,
+        text: &(impl AsRef<[u8]> + ?Sized),
+    ) {
+        self.replace_text_impl(y, origin_x, clip_right, text.as_ref(), true);
     }
 
     fn replace_text_impl(
@@ -213,9 +228,14 @@ impl Framebuffer {
         origin_x: CoordType,
         clip_right: CoordType,
         text: &[u8],
+        highlight_unusual_whitespace: bool,
     ) {
         let scratch = scratch_arena(None);
-        let sanitized = sanitize_control_chars(&scratch, text);
+        let sanitized = if highlight_unusual_whitespace {
+            sanitize_control_chars_with_unusual_whitespace(&scratch, text)
+        } else {
+            sanitize_control_chars(&scratch, text)
+        };
 
         let back = &mut self.buffers[self.frame_counter & 1];
         back.text.replace_text(y, origin_x, clip_right, &sanitized);
@@ -225,7 +245,7 @@ impl Framebuffer {
         }
     }
 
-    /// Highlights the replacements that [`sanitize_control_chars`] made in yellow.
+    /// Highlights the sanitizer's visual replacements in yellow.
     #[cold]
     fn highlight_sanitized(
         &mut self,
@@ -987,3 +1007,89 @@ impl Cursor {
         Self { pos: Point { x: -1, y: -1 }, overtype: false }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::buffer::TextBuffer;
+
+    #[test]
+    #[allow(clippy::single_range_in_vec_init)]
+    fn unusual_whitespace_visualization_is_opt_in_and_preserves_width() {
+        fn width(text: &str) -> CoordType {
+            let bytes = text.as_bytes();
+            let mut cfg = MeasurementConfig::new(&bytes);
+            cfg.goto_offset(bytes.len()).visual_pos.x
+        }
+
+        const CODEPOINTS: &[u32] = &[
+            0x00A0, 0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008,
+            0x2009, 0x200A, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000,
+        ];
+
+        for &codepoint in CODEPOINTS {
+            let ch = char::from_u32(codepoint).unwrap();
+            let mut input_buffer = [0; 4];
+            let input = ch.encode_utf8(&mut input_buffer);
+            let scratch = scratch_arena(None);
+            let unchanged = sanitize_control_chars(&scratch, input);
+            let sanitized = sanitize_control_chars_with_unusual_whitespace(&scratch, input);
+            let expected = match ch {
+                '\u{2028}' | '\u{2029}' => "\u{2424}",
+                '\u{3000}' => "\u{2423} ",
+                _ => "\u{2423}",
+            };
+
+            assert!(matches!(&unchanged, MaybeOwned::Borrowed(_)), "U+{codepoint:04X}");
+            assert_eq!(&*unchanged, input, "U+{codepoint:04X}");
+            let MaybeOwned::Owned(sanitized) = &sanitized else {
+                panic!("U+{codepoint:04X} was not visualized");
+            };
+            assert_eq!(&*sanitized.text, expected, "U+{codepoint:04X}");
+            assert_eq!(
+                sanitized.unsane_ranges.as_slice(),
+                &[0..expected.len()],
+                "U+{codepoint:04X}"
+            );
+            assert_eq!(width(input), width(&sanitized.text), "U+{codepoint:04X}");
+        }
+
+        const NON_TARGETS: &str = " \u{00A1}\u{167F}\u{1681}\u{180E}\u{1FFF}\u{200B}\u{2027}\
+            \u{202A}\u{202E}\u{2030}\u{205E}\u{2060}\u{2FFF}\u{3001}\u{FEFF}";
+        let scratch = scratch_arena(None);
+        let sanitized = sanitize_control_chars_with_unusual_whitespace(&scratch, NON_TARGETS);
+
+        assert!(matches!(&sanitized, MaybeOwned::Borrowed(_)));
+        assert_eq!(&*sanitized, NON_TARGETS);
+    }
+
+    #[test]
+    fn text_selection_takes_precedence_over_unusual_whitespace_highlight() {
+        let mut text_buffer = TextBuffer::new(true).unwrap();
+        text_buffer.write_canon("\u{00A0}".as_bytes());
+        text_buffer.select_all();
+        text_buffer.set_width(4);
+        text_buffer.set_unusual_whitespace_highlight_enabled(true);
+
+        let mut framebuffer = Framebuffer::new();
+        framebuffer.flip(Size { width: 4, height: 1 });
+        assert!(
+            text_buffer
+                .render(
+                    Point::default(),
+                    Rect { left: 0, top: 0, right: 4, bottom: 1 },
+                    true,
+                    &mut framebuffer,
+                )
+                .is_some()
+        );
+
+        let expected_selection_bg = framebuffer
+            .indexed(IndexedColor::Foreground)
+            .oklab_blend(framebuffer.indexed_alpha(IndexedColor::BrightBlue, 1, 2));
+        let back = &framebuffer.buffers[framebuffer.frame_counter & 1];
+
+        assert!(back.text.lines[0].starts_with("\u{2423}"));
+        assert_eq!(back.bg_bitmap.data[0], expected_selection_bg);
+    }
+}

crates/edit/src/tui.rs

@@ -2749,6 +2749,12 @@ impl<'a> Context<'a, '_> {
                     kbmod::CTRL => tb.delete(CursorMovement::Word, -1),
                     _ => return false,
                 },
+                vk::I => match modifiers {
+                    kbmod::ALT => tb.set_unusual_whitespace_highlight_enabled(
+                        !tb.is_unusual_whitespace_highlight_enabled(),
+                    ),
+                    _ => return false,
+                },
                 vk::L => match modifiers {
                     kbmod::CTRL => tb.select_line(),
                     _ => return false,
@@ -4110,3 +4116,39 @@ impl<'a> Node<'a> {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn alt_i_toggles_unusual_whitespace_highlighting() -> io::Result<()> {
+        let mut tui = Tui::new()?;
+        let buffer = TextBuffer::new_rc(true)?;
+        let mut content = TextareaContent {
+            buffer: &buffer,
+            scroll_offset: Point::default(),
+            scroll_offset_y_drag_start: CoordType::MIN,
+            scroll_offset_x_max: 0,
+            thumb_height: 0,
+            preferred_column: 0,
+            single_line: false,
+            has_focus: true,
+        };
+        let node = Node::default();
+
+        {
+            let mut ctx = tui.create_context(Some(Input::Keyboard(vk::I)));
+            assert!(!ctx.textarea_handle_input(&mut content, &node, false));
+        }
+        assert!(!buffer.borrow().is_unusual_whitespace_highlight_enabled());
+
+        for expected in [true, false] {
+            let mut ctx = tui.create_context(Some(Input::Keyboard(kbmod::ALT | vk::I)));
+            assert!(ctx.textarea_handle_input(&mut content, &node, false));
+            assert_eq!(buffer.borrow().is_unusual_whitespace_highlight_enabled(), expected);
+        }
+
+        Ok(())
+    }
+}