tests: add whitespace tests for vertical tab behavior

tests/ui/README.md

@@ -1582,6 +1582,8 @@ Tests on various well-formedness checks, e.g. [Type-checking normal functions](h
 
 Tests on `where` clauses. See [Where clauses | Reference](https://doc.rust-lang.org/reference/items/generics.html#where-clauses).
 
+## `whitespace`
+
 ## `tests/ui/windows-subsystem/`: `#![windows_subsystem = ""]`
 
 See [the `windows_subsystem` attribute](https://doc.rust-lang.org/reference/runtime.html#the-windows_subsystem-attribute).

tests/ui/whitespace/ascii_whitespace_excludes_vertical_tab.rs

@@ -0,0 +1,22 @@
+// This test checks that split_ascii_whitespace does NOT split on
+// vertical tab (\x0B), because the standard library uses the WhatWG
+// Infra Standard definition of ASCII whitespace, which excludes
+// vertical tab.
+//
+// See: https://github.com/rust-lang/rust-project-goals/issues/53
+
+fn main() {
+    let s = "a\x0Bb";
+
+    let parts: Vec<&str> = s.split_ascii_whitespace().collect();
+
+    assert_eq!(parts.len(), 1,
+        "vertical tab should not be treated as ASCII whitespace");
+
+    let s2 = "a b";
+    let parts2: Vec<&str> = s2.split_ascii_whitespace().collect();
+    assert_eq!(parts2.len(), 2,
+        "regular space should split correctly");
+
+}
+

tests/ui/whitespace/vertical_tab_lexer.rs

@@ -0,0 +1,13 @@
+// This test checks that the Rust lexer treats vertical tab (\x0B) as
+// whitespace, consistent with Unicode Pattern_White_Space. Note that
+// the standard library's is_ascii_whitespace does NOT include vertical
+// tab, following the WhatWG Infra Standard instead.
+//
+// See: https://github.com/rust-lang/rust-project-goals/issues/53
+
+fn main() {
+    let x = 5;
+    let y = 10;
+    let z = x + y;
+
+}