Lexer: unicode tests

nois-lang · Jun 13, 2023 · 5ca9fc2 · 5ca9fc2
1 parent 7592b0b
commit 5ca9fc2
Showing 1 changed file with 16 additions and 0 deletions.
diff --git a/src/lexer/lexer.spec.ts b/src/lexer/lexer.spec.ts
@@ -95,6 +95,14 @@ let main = (): Unit {
             ])
         })
 
+        it('unicode', () => {
+            expect(tokenize(`'\\u{1}' '\\u{ffff}'`)).toEqual([
+                { kind: 'char', value: `'\\u{1}'`, location: { start: 0, end: 6 } },
+                { kind: 'char', value: `'\\u{ffff}'`, location: { start: 8, end: 17 } },
+                { kind: 'eof', value: '', location: { start: 18, end: 18 } }
+            ])
+        })
+
         it('unterminated', () => {
             expect(tokenize(`'h`)).toEqual([
                 { kind: 'unterminated-char', value: `'h`, location: { start: 0, end: 2 } },
@@ -133,6 +141,14 @@ let main = (): Unit {
             ])
         })
 
+        it('unicode', () => {
+            expect(tokenize(`"\\u{1} \\u{ffff}"`)).toEqual([
+                { kind: 'string', value: `"\\u{1} \\u{ffff}"`, location: { start: 0, end: 15 } },
+                { kind: 'eof', value: '', location: { start: 16, end: 16 } }
+            ])
+        })
+
+
         it('unterminated', () => {
             expect(tokenize(`"string 123 ok\n`)).toEqual([
                 { kind: 'unterminated-string', value: `"string 123 ok`, location: { start: 0, end: 13 } },