Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed Windows Terminal UTF-16 surrogate pair issue #857

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 49 additions & 13 deletions src/event/sys/windows/parse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,19 +61,16 @@ pub(crate) fn handle_key_event(
}

fn handle_surrogate(surrogate_buffer: &mut Option<u16>, new_surrogate: u16) -> Option<char> {
kazatsuyu marked this conversation as resolved.
Show resolved Hide resolved
match *surrogate_buffer {
Some(buffered_surrogate) => {
*surrogate_buffer = None;
std::char::decode_utf16([buffered_surrogate, new_surrogate])
.next()
.unwrap()
.ok()
}
None => {
*surrogate_buffer = Some(new_surrogate);
None
}
if new_surrogate < LOW_SURROGATE_FIRST {
// Discard any buffered surrogate value if another high surrogate comes.
*surrogate_buffer = Some(new_surrogate);
return None;
}
let buffered_surrogate = surrogate_buffer.take()?;
std::char::decode_utf16([buffered_surrogate, new_surrogate])
.next()
.unwrap()
.ok()
}

impl From<&ControlKeyState> for KeyModifiers {
Expand Down Expand Up @@ -268,7 +265,7 @@ fn parse_key_event_record(key_event: &KeyEventRecord) -> Option<WindowsKeyEvent>
// are handled by their virtual key codes above.
get_char_for_key(key_event).map(KeyCode::Char)
}
surrogate @ 0xD800..=0xDFFF => {
surrogate @ HIGH_SURROGATE_FIRST..=LOW_SURROGATE_LAST => {
return Some(WindowsKeyEvent::Surrogate(surrogate));
}
unicode_scalar_value => {
Expand Down Expand Up @@ -376,3 +373,42 @@ fn parse_mouse_event_record(
modifiers,
}))
}

const HIGH_SURROGATE_FIRST: u16 = 0xD800;
const LOW_SURROGATE_FIRST: u16 = 0xDC00;
const LOW_SURROGATE_LAST: u16 = 0xDFFF;
Comment on lines +377 to +379
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Constants generally should be at the top of the file


#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_handle_surrogate() {
fn testcase(mut buf: Option<u16>, new: u16) -> (Option<u16>, Option<char>) {
let ch = handle_surrogate(&mut buf, new);
(buf, ch)
}
assert_eq!(
[
// Continuous high surrogates
testcase(Some(0xD800), 0xDBFF),
// A normal surrogate pair
testcase(Some(0xD800), 0xDC00),
// An empty buffer and a high surrogate
testcase(None, 0xDBFF),
// An empty buffer and a low surrogate
testcase(None, 0xDC00),
],
[
// The newer one will be stored
(Some(0xDBFF), None),
// A valid char will return
(None, Some('\u{10000}')),
// The high surrogate will be stored
(Some(0xDBFF), None),
// The low surrogate will be ignored
(None, None),
]
);
Comment on lines +388 to +412
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Simplify (individual assertions, no testcase function, descriptions on the assertions, arranged in order that makes most sense (empty to full)):

Suggested change
let ch = handle_surrogate(&mut buf, new);
(buf, ch)
}
assert_eq!(
[
// Continuous high surrogates
testcase(Some(0xD800), 0xDBFF),
// A normal surrogate pair
testcase(Some(0xD800), 0xDC00),
// An empty buffer and a high surrogate
testcase(None, 0xDBFF),
// An empty buffer and a low surrogate
testcase(None, 0xDC00),
],
[
// The newer one will be stored
(Some(0xDBFF), None),
// A valid char will return
(None, Some('\u{10000}')),
// The high surrogate will be stored
(Some(0xDBFF), None),
// The low surrogate will be ignored
(None, None),
]
);
let mut buf = None;
let next = handle_surrogate(&mut buf, 0xDC00);
assert_eq!(
(buf, next),
(None, None),
"Unexpected low surrogate is ignored"
);
let mut buf = None;
let next = handle_surrogate(&mut buf, 0xDBFF);
assert_eq!(
(buf, next),
(Some(0xDBFF), None),
"High surrogate is buffered"
);
let mut buf = Some(0xD800);
let next = handle_surrogate(&mut buf, 0xDBFF);
assert_eq!(
(buf, next),
(Some(0xDBFF), None),
"Consecutive high surrogates"
);
let mut buf = Some(0xD800);
let next = handle_surrogate(&mut buf, 0xDC00);
assert_eq!(
(buf, next),
(None, Some('\u{10000}')),
"Valid Surrogate pair"
);

}
}