diff --git a/crates/edit/src/bin/edit/draw_menubar.rs b/crates/edit/src/bin/edit/draw_menubar.rs index 401614786a1..44e8c0911b2 100644 --- a/crates/edit/src/bin/edit/draw_menubar.rs +++ b/crates/edit/src/bin/edit/draw_menubar.rs @@ -140,6 +140,16 @@ fn draw_menu_view(ctx: &mut Context, state: &mut State) { tb.set_word_wrap(!word_wrap); ctx.needs_rerender(); } + let highlight_unusual_whitespace = tb.is_unusual_whitespace_highlight_enabled(); + if ctx.menubar_menu_checkbox( + loc(LocId::ViewHighlightUnusualWhitespace), + 'I', + kbmod::ALT | vk::I, + highlight_unusual_whitespace, + ) { + tb.set_unusual_whitespace_highlight_enabled(!highlight_unusual_whitespace); + ctx.needs_rerender(); + } } ctx.menubar_menu_end(); diff --git a/crates/edit/src/buffer/mod.rs b/crates/edit/src/buffer/mod.rs index 777501f774a..4f565802616 100644 --- a/crates/edit/src/buffer/mod.rs +++ b/crates/edit/src/buffer/mod.rs @@ -258,6 +258,7 @@ pub struct TextBuffer { margin_enabled: bool, word_wrap_column: CoordType, word_wrap_enabled: bool, + unusual_whitespace_highlight_enabled: bool, tab_size: CoordType, indent_with_tabs: bool, line_highlight_enabled: bool, @@ -308,6 +309,7 @@ impl TextBuffer { margin_enabled: false, word_wrap_column: 0, word_wrap_enabled: false, + unusual_whitespace_highlight_enabled: false, tab_size: 4, indent_with_tabs: false, line_highlight_enabled: false, @@ -538,6 +540,19 @@ impl TextBuffer { } } + /// Is highlighting of unusual Unicode whitespace enabled? + pub fn is_unusual_whitespace_highlight_enabled(&self) -> bool { + self.unusual_whitespace_highlight_enabled + } + + /// Enable or disable highlighting of unusual Unicode whitespace. + /// + /// This only affects how characters are visualized (a stand-in glyph plus a + /// yellow background), not the layout, so no reflow is necessary. + pub fn set_unusual_whitespace_highlight_enabled(&mut self, enabled: bool) { + self.unusual_whitespace_highlight_enabled = enabled; + } + /// Set the width available for layout. /// /// Ideally this would be a pure UI concern, but the text buffer needs this @@ -1790,6 +1805,7 @@ impl TextBuffer { let height = destination.height(); let line_number_width = self.margin_width.max(3) as usize - 3; let text_width = width - self.margin_width; + let highlight_unusual_whitespace = self.unusual_whitespace_highlight_enabled; let mut visual_pos_x_max = 0; // Pick the cursor closer to the `origin.y`. @@ -1869,6 +1885,7 @@ impl TextBuffer { } let mut selection_off = 0..0; + let mut selection_highlight = None; // Figure out the selection range on this line, if any. if cursor_beg.visual_pos.y == visual_line @@ -1919,8 +1936,12 @@ impl TextBuffer { bg = bg.oklab_blend(fb.indexed_alpha(IndexedColor::Background, 1, 2)); }; let fg = fb.contrasted(bg); - fb.blend_bg(rect, bg); - fb.blend_fg(rect, fg); + if highlight_unusual_whitespace { + selection_highlight = Some((rect, bg, fg)); + } else { + fb.blend_bg(rect, bg); + fb.blend_fg(rect, fg); + } } // Nothing to do if the entire line is empty. @@ -1965,7 +1986,7 @@ impl TextBuffer { let mut whitespace = TAB_WHITESPACE; let mut prefix_add = 0; - if is_tab || visualize { + if is_tab || (visualize && !highlight_unusual_whitespace) { // We need the character's visual position in order to either compute the tab size, // or set the foreground color of the visualizer, respectively. // TODO: Doing this char-by-char is bad for performance. @@ -1985,19 +2006,22 @@ impl TextBuffer { (VISUAL_SPACE, VISUAL_SPACE_PREFIX_ADD) }; - // Make the visualized characters slightly gray. - let visualizer_rect = { - let left = destination.left - + self.margin_width - + cursor_line.visual_pos.x - - origin.x; - let top = destination.top + cursor_line.visual_pos.y - origin.y; - Rect { left, top, right: left + 1, bottom: top + 1 } - }; - fb.blend_fg( - visualizer_rect, - fb.indexed_alpha(IndexedColor::Foreground, 1, 2), - ); + if !highlight_unusual_whitespace { + // Make the visualized characters slightly gray. + let visualizer_rect = { + let left = destination.left + + self.margin_width + + cursor_line.visual_pos.x + - origin.x; + let top = + destination.top + cursor_line.visual_pos.y - origin.y; + Rect { left, top, right: left + 1, bottom: top + 1 } + }; + fb.blend_fg( + visualizer_rect, + fb.indexed_alpha(IndexedColor::Foreground, 1, 2), + ); + } } line.extend_from_slice( @@ -2014,7 +2038,23 @@ impl TextBuffer { visual_pos_x_max = visual_pos_x_max.max(cursor_end.visual_pos.x); } - fb.replace_text(destination.top + y, destination.left, destination.right, &line); + if highlight_unusual_whitespace { + fb.replace_text_with_unusual_whitespace_highlight( + destination.top + y, + destination.left, + destination.right, + &line, + ); + } else { + fb.replace_text(destination.top + y, destination.left, destination.right, &line); + } + + // Selection takes precedence over sanitizer highlights, so unusual + // whitespace remains visibly selected instead of staying solid yellow. + if let Some((rect, bg, fg)) = selection_highlight { + fb.blend_bg(rect, bg); + fb.blend_fg(rect, fg); + } cursor = cursor_end; } diff --git a/crates/edit/src/framebuffer.rs b/crates/edit/src/framebuffer.rs index 74562af98d0..b3bc7154cf0 100644 --- a/crates/edit/src/framebuffer.rs +++ b/crates/edit/src/framebuffer.rs @@ -11,7 +11,9 @@ use std::slice::ChunksExact; use stdext::arena::{Arena, scratch_arena}; use stdext::collections::BString; use stdext::simd::memset; -use stdext::unicode::{SanitizedControlChars, sanitize_control_chars}; +use stdext::unicode::{ + SanitizedControlChars, sanitize_control_chars, sanitize_control_chars_with_unusual_whitespace, +}; use stdext::{MaybeOwned, arena_write_fmt}; use crate::helpers::{CoordType, Point, Rect, Size}; @@ -195,7 +197,7 @@ impl Framebuffer { /// Replaces text contents in a single line of the framebuffer. /// All coordinates are in viewport coordinates. - /// Assumes that control characters have been replaced or escaped. + /// Control characters and invalid UTF-8 are visualized before rendering. #[inline] pub fn replace_text( &mut self, @@ -204,7 +206,20 @@ impl Framebuffer { clip_right: CoordType, text: &(impl AsRef<[u8]> + ?Sized), ) { - self.replace_text_impl(y, origin_x, clip_right, text.as_ref()); + self.replace_text_impl(y, origin_x, clip_right, text.as_ref(), false); + } + + /// Replaces text and highlights unusual Unicode whitespace in a single line. + /// All coordinates are in viewport coordinates. + #[inline] + pub fn replace_text_with_unusual_whitespace_highlight( + &mut self, + y: CoordType, + origin_x: CoordType, + clip_right: CoordType, + text: &(impl AsRef<[u8]> + ?Sized), + ) { + self.replace_text_impl(y, origin_x, clip_right, text.as_ref(), true); } fn replace_text_impl( @@ -213,9 +228,14 @@ impl Framebuffer { origin_x: CoordType, clip_right: CoordType, text: &[u8], + highlight_unusual_whitespace: bool, ) { let scratch = scratch_arena(None); - let sanitized = sanitize_control_chars(&scratch, text); + let sanitized = if highlight_unusual_whitespace { + sanitize_control_chars_with_unusual_whitespace(&scratch, text) + } else { + sanitize_control_chars(&scratch, text) + }; let back = &mut self.buffers[self.frame_counter & 1]; back.text.replace_text(y, origin_x, clip_right, &sanitized); @@ -225,7 +245,7 @@ impl Framebuffer { } } - /// Highlights the replacements that [`sanitize_control_chars`] made in yellow. + /// Highlights the sanitizer's visual replacements in yellow. #[cold] fn highlight_sanitized( &mut self, @@ -987,3 +1007,89 @@ impl Cursor { Self { pos: Point { x: -1, y: -1 }, overtype: false } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::buffer::TextBuffer; + + #[test] + #[allow(clippy::single_range_in_vec_init)] + fn unusual_whitespace_visualization_is_opt_in_and_preserves_width() { + fn width(text: &str) -> CoordType { + let bytes = text.as_bytes(); + let mut cfg = MeasurementConfig::new(&bytes); + cfg.goto_offset(bytes.len()).visual_pos.x + } + + const CODEPOINTS: &[u32] = &[ + 0x00A0, 0x1680, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, + 0x2009, 0x200A, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000, + ]; + + for &codepoint in CODEPOINTS { + let ch = char::from_u32(codepoint).unwrap(); + let mut input_buffer = [0; 4]; + let input = ch.encode_utf8(&mut input_buffer); + let scratch = scratch_arena(None); + let unchanged = sanitize_control_chars(&scratch, input); + let sanitized = sanitize_control_chars_with_unusual_whitespace(&scratch, input); + let expected = match ch { + '\u{2028}' | '\u{2029}' => "\u{2424}", + '\u{3000}' => "\u{2423} ", + _ => "\u{2423}", + }; + + assert!(matches!(&unchanged, MaybeOwned::Borrowed(_)), "U+{codepoint:04X}"); + assert_eq!(&*unchanged, input, "U+{codepoint:04X}"); + let MaybeOwned::Owned(sanitized) = &sanitized else { + panic!("U+{codepoint:04X} was not visualized"); + }; + assert_eq!(&*sanitized.text, expected, "U+{codepoint:04X}"); + assert_eq!( + sanitized.unsane_ranges.as_slice(), + &[0..expected.len()], + "U+{codepoint:04X}" + ); + assert_eq!(width(input), width(&sanitized.text), "U+{codepoint:04X}"); + } + + const NON_TARGETS: &str = " \u{00A1}\u{167F}\u{1681}\u{180E}\u{1FFF}\u{200B}\u{2027}\ + \u{202A}\u{202E}\u{2030}\u{205E}\u{2060}\u{2FFF}\u{3001}\u{FEFF}"; + let scratch = scratch_arena(None); + let sanitized = sanitize_control_chars_with_unusual_whitespace(&scratch, NON_TARGETS); + + assert!(matches!(&sanitized, MaybeOwned::Borrowed(_))); + assert_eq!(&*sanitized, NON_TARGETS); + } + + #[test] + fn text_selection_takes_precedence_over_unusual_whitespace_highlight() { + let mut text_buffer = TextBuffer::new(true).unwrap(); + text_buffer.write_canon("\u{00A0}".as_bytes()); + text_buffer.select_all(); + text_buffer.set_width(4); + text_buffer.set_unusual_whitespace_highlight_enabled(true); + + let mut framebuffer = Framebuffer::new(); + framebuffer.flip(Size { width: 4, height: 1 }); + assert!( + text_buffer + .render( + Point::default(), + Rect { left: 0, top: 0, right: 4, bottom: 1 }, + true, + &mut framebuffer, + ) + .is_some() + ); + + let expected_selection_bg = framebuffer + .indexed(IndexedColor::Foreground) + .oklab_blend(framebuffer.indexed_alpha(IndexedColor::BrightBlue, 1, 2)); + let back = &framebuffer.buffers[framebuffer.frame_counter & 1]; + + assert!(back.text.lines[0].starts_with("\u{2423}")); + assert_eq!(back.bg_bitmap.data[0], expected_selection_bg); + } +} diff --git a/crates/edit/src/tui.rs b/crates/edit/src/tui.rs index 9826c91cfc7..eb3fc938cfb 100644 --- a/crates/edit/src/tui.rs +++ b/crates/edit/src/tui.rs @@ -2749,6 +2749,12 @@ impl<'a> Context<'a, '_> { kbmod::CTRL => tb.delete(CursorMovement::Word, -1), _ => return false, }, + vk::I => match modifiers { + kbmod::ALT => tb.set_unusual_whitespace_highlight_enabled( + !tb.is_unusual_whitespace_highlight_enabled(), + ), + _ => return false, + }, vk::L => match modifiers { kbmod::CTRL => tb.select_line(), _ => return false, @@ -4110,3 +4116,39 @@ impl<'a> Node<'a> { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn alt_i_toggles_unusual_whitespace_highlighting() -> io::Result<()> { + let mut tui = Tui::new()?; + let buffer = TextBuffer::new_rc(true)?; + let mut content = TextareaContent { + buffer: &buffer, + scroll_offset: Point::default(), + scroll_offset_y_drag_start: CoordType::MIN, + scroll_offset_x_max: 0, + thumb_height: 0, + preferred_column: 0, + single_line: false, + has_focus: true, + }; + let node = Node::default(); + + { + let mut ctx = tui.create_context(Some(Input::Keyboard(vk::I))); + assert!(!ctx.textarea_handle_input(&mut content, &node, false)); + } + assert!(!buffer.borrow().is_unusual_whitespace_highlight_enabled()); + + for expected in [true, false] { + let mut ctx = tui.create_context(Some(Input::Keyboard(kbmod::ALT | vk::I))); + assert!(ctx.textarea_handle_input(&mut content, &node, false)); + assert_eq!(buffer.borrow().is_unusual_whitespace_highlight_enabled(), expected); + } + + Ok(()) + } +} diff --git a/crates/stdext/src/unicode/sanitize.rs b/crates/stdext/src/unicode/sanitize.rs index ce93f179947..ac629078f9b 100644 --- a/crates/stdext/src/unicode/sanitize.rs +++ b/crates/stdext/src/unicode/sanitize.rs @@ -7,9 +7,9 @@ use crate::collections::{BString, BVec}; use crate::unicode::Utf8Chars; pub struct SanitizedControlChars<'a> { - /// Sanitized string with all C0/C1 control characters replaced by their Unicode representations. + /// Sanitized string with visual replacements for unsafe or unusual characters. pub text: BString<'a>, - /// Byte ranges of the replacement characters within [`Self::text`]. + /// Byte ranges of the visual replacements within [`Self::text`]. pub unsane_ranges: BVec<'a, Range>, } @@ -19,7 +19,7 @@ impl Borrow for SanitizedControlChars<'_> { } } -/// Strips all C0/C1 control characters and invalid UTF8 from the text. +/// Strips all C0/C1 control characters and invalid UTF-8 from the text. #[inline] pub fn sanitize_control_chars<'a>( arena: &'a Arena, @@ -28,22 +28,77 @@ pub fn sanitize_control_chars<'a>( sanitize_control_chars_impl(arena, text.as_ref()) } -/// Strips all C0/C1 control characters and invalid UTF8 from the text. +/// Sanitizes control characters and visualizes unusual Unicode whitespace. +#[inline] +pub fn sanitize_control_chars_with_unusual_whitespace<'a>( + arena: &'a Arena, + text: &'a (impl AsRef<[u8]> + ?Sized), +) -> MaybeOwned<'a, str, SanitizedControlChars<'a>> { + sanitize_control_chars_impl_with_options::(arena, text.as_ref()) +} + +/// Strips all C0/C1 control characters and invalid UTF-8 from the text. pub fn sanitize_control_chars_impl<'a>( arena: &'a Arena, text: &'a [u8], +) -> MaybeOwned<'a, str, SanitizedControlChars<'a>> { + sanitize_control_chars_impl_with_options::(arena, text) +} + +fn sanitize_control_chars_impl_with_options<'a, const VISUALIZE_UNUSUAL_WHITESPACE: bool>( + arena: &'a Arena, + text: &'a [u8], ) -> MaybeOwned<'a, str, SanitizedControlChars<'a>> { #[inline(always)] - fn is_unsane(text: &[u8], beg: usize, end: usize, ch: char) -> bool { + fn is_unsane( + text: &[u8], + beg: usize, + end: usize, + ch: char, + ) -> bool { // Utf8Chars yields U+FFFD for invalid inputs, but it can also be a legitimate source character. - // So, we need to check if the original bytes were actually invalid UTF8 (slow path). + // So, we need to check if the original bytes were actually invalid UTF-8 (slow path). #[cold] fn is_invalid_utf8(text: &[u8], beg: usize, end: usize) -> bool { &text[beg..end] != "\u{FFFD}".as_bytes() } - ch < '\x20' + + if ch < '\x20' || ('\u{7F}'..='\u{9F}').contains(&ch) || (ch == char::REPLACEMENT_CHARACTER && is_invalid_utf8(text, beg, end)) + { + return true; + } + + VISUALIZE_UNUSUAL_WHITESPACE && is_unusual_unicode_whitespace(ch) + } + + // Non-ASCII characters with the Unicode White_Space property, excluding controls + // which are already handled above. The stand-ins preserve their terminal width. + #[inline(always)] + fn is_unusual_unicode_whitespace(ch: char) -> bool { + ('\u{2000}'..='\u{200A}').contains(&ch) // EN QUAD through HAIR SPACE + || matches!( + ch, + '\u{00A0}' // NO-BREAK SPACE + | '\u{1680}' // OGHAM SPACE MARK + | '\u{2028}' // LINE SEPARATOR + | '\u{2029}' // PARAGRAPH SEPARATOR + | '\u{202F}' // NARROW NO-BREAK SPACE + | '\u{205F}' // MEDIUM MATHEMATICAL SPACE + | '\u{3000}' // IDEOGRAPHIC SPACE + ) + } + + // U+2423 = SYMBOL FOR SPACE, U+2424 = SYMBOL FOR NEWLINE. U+3000 is two cells + // wide, so its replacement includes a trailing space to preserve the layout. + #[inline(always)] + fn unusual_whitespace_standin(ch: char) -> &'static str { + match ch { + '\u{2028}' | '\u{2029}' => "\u{2424}", + '\u{3000}' => "\u{2423} ", + _ => "\u{2423}", + } } if text.is_empty() { @@ -67,7 +122,7 @@ pub fn sanitize_control_chars_impl<'a>( Some(ch) => ch, None => break, }; - if is_unsane(text, sane_end, chars.offset(), ch) { + if is_unsane::(text, sane_end, chars.offset(), ch) { break; } } @@ -91,21 +146,24 @@ pub fn sanitize_control_chars_impl<'a>( // Copy and sanitize as many characters as necessary. let unsane_beg = sanitized.text.len(); loop { - // Append a Unicode representation of the C0 or C1 control character. - let mut visualized = "\u{FFFD}"; - - if ch != '\u{FFFD}' { - visualizer_buf[2] = if ch <= '\x1f' { - 0x80 | ch as u8 // U+2400..=U+241F - } else if ch == '\x7f' { - 0xA1 // U+2421 + let visualized: &str = + if VISUALIZE_UNUSUAL_WHITESPACE && is_unusual_unicode_whitespace(ch) { + unusual_whitespace_standin(ch) + } else if ch == '\u{FFFD}' { + // Invalid UTF-8 is visualized as U+FFFD. + "\u{FFFD}" } else { - // NOTE: Unicode says to use U+FFFD, but that one is ambiguous width. - 0xA6 // U+2426, because there are no pictures for C1 control characters. + visualizer_buf[2] = if ch <= '\x1f' { + 0x80 | ch as u8 // U+2400..=U+241F + } else if ch == '\x7f' { + 0xA1 // U+2421 + } else { + // NOTE: Unicode says to use U+FFFD, but that one is ambiguous width. + 0xA6 // U+2426, because there are no pictures for C1 control characters. + }; + // Our manually constructed UTF-8 is never going to be invalid. Trust. + unsafe { std::str::from_utf8_unchecked(&visualizer_buf) } }; - // Our manually constructed UTF8 is never going to be invalid. Trust. - visualized = unsafe { std::str::from_utf8_unchecked(&visualizer_buf) }; - } sanitized.text.push_str(arena, visualized); @@ -115,7 +173,7 @@ pub fn sanitize_control_chars_impl<'a>( Some(ch) => ch, None => break, }; - if !is_unsane(text, sane_beg, chars.offset(), ch) { + if !is_unsane::(text, sane_beg, chars.offset(), ch) { break; } } @@ -140,19 +198,34 @@ mod tests { Owned(&'a str, &'a [Range]), } - const TESTS: &[(&[u8], Result)] = &[ - (b"", Result::Borrowed("")), - ("aé".as_bytes(), Result::Borrowed("aé")), - ("a\u{FFFD}b".as_bytes(), Result::Borrowed("a\u{FFFD}b")), - ("aé\u{1}\u{7f}\u{9f}b".as_bytes(), Result::Owned("aé␁␡␦b", &[3..12])), - ("\u{1}a\0".as_bytes(), Result::Owned("␁a␀", &[0..3, 4..7])), - (b"a\xff\xffb", Result::Owned("a\u{FFFD}\u{FFFD}b", &[1..7])), - (b"\xf0\x9f\x98", Result::Owned("\u{FFFD}", &[0..3])), + const TESTS: &[(&[u8], bool, Result)] = &[ + (b"", false, Result::Borrowed("")), + ("aé".as_bytes(), false, Result::Borrowed("aé")), + ("a\u{FFFD}b".as_bytes(), false, Result::Borrowed("a\u{FFFD}b")), + ("aé\u{1}\u{7f}\u{9f}b".as_bytes(), false, Result::Owned("aé␁␡␦b", &[3..12])), + ("\u{1}a\0".as_bytes(), false, Result::Owned("␁a␀", &[0..3, 4..7])), + (b"a\xff\xffb", false, Result::Owned("a\u{FFFD}\u{FFFD}b", &[1..7])), + (b"\xf0\x9f\x98", false, Result::Owned("\u{FFFD}", &[0..3])), + // Unusual whitespace visualization is opt-in. + ("a\u{00A0}b".as_bytes(), false, Result::Borrowed("a\u{00A0}b")), + ("a\u{00A0}b".as_bytes(), true, Result::Owned("a\u{2423}b", &[1..4])), + ("a\u{2007}b".as_bytes(), true, Result::Owned("a\u{2423}b", &[1..4])), + ("a\u{2029}b".as_bytes(), true, Result::Owned("a\u{2424}b", &[1..4])), + ("a\u{3000}b".as_bytes(), true, Result::Owned("a\u{2423} b", &[1..5])), + ( + "\u{00A0}\u{2028}\u{3000}".as_bytes(), + true, + Result::Owned("\u{2423}\u{2424}\u{2423} ", &[0..10]), + ), ]; - for (test, expected) in TESTS { + for (test, visualize_unusual_whitespace, expected) in TESTS { let scratch = scratch_arena(None); - let actual = sanitize_control_chars(&scratch, test); + let actual = if *visualize_unusual_whitespace { + sanitize_control_chars_with_unusual_whitespace(&scratch, test) + } else { + sanitize_control_chars(&scratch, test) + }; let actual = match &actual { MaybeOwned::Borrowed(b) => Result::Borrowed(b), MaybeOwned::Owned(o) => Result::Owned(&o.text, &o.unsane_ranges), diff --git a/i18n/edit.toml b/i18n/edit.toml index 11039636308..a7ca25d2c89 100644 --- a/i18n/edit.toml +++ b/i18n/edit.toml @@ -999,6 +999,11 @@ vi = "Ngắt dòng tự động" zh-hans = "自动换行" zh-hant = "自動換行" +[ViewHighlightUnusualWhitespace] +en = "Highlight Unusual Whitespace" +zh-hans = "高亮特殊空白字符" +zh-hant = "高亮特殊空白字元" + [ViewGoToFile] en = "Go to File…" ar = "الانتقال إلى ملف…"