Lexer: erroneous token kinds; unterminated literals

nois-lang · Jun 11, 2023 · 1658fa4 · 1658fa4
1 parent c02de72
commit 1658fa4
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 10 deletions.
diff --git a/src/error.ts b/src/error.ts
@@ -10,7 +10,7 @@ export interface SyntaxError {
 }
 
 export const prettyLexerError = (token: ParseToken): string => {
-    return red(`lexer error: unknown token \`${token.value}\``)
+    return red(`lexer error: ${token.kind} token \`${token.value}\``)
 }
 
 export const prettySyntaxError = (error: SyntaxError): string => {

diff --git a/src/index.ts b/src/index.ts
@@ -1,4 +1,4 @@
-import { tokenize } from './lexer/lexer'
+import { erroneousTokenKinds, tokenize } from './lexer/lexer'
 import { existsSync, readFileSync } from 'fs'
 import { join, resolve } from 'path'
 import { compactNode, Parser } from './parser/parser'
@@ -26,9 +26,9 @@ if (!existsSync(sourcePath)) {
 const source = { str: readFileSync(sourcePath).toString(), filename: path }
 
 const tokens = tokenize(source.str)
-const unknownTokens = tokens.filter(t => t.kind === 'unknown')
-if (unknownTokens.length > 0) {
-    for (const t of unknownTokens) {
+const errorTokens = tokens.filter(t => erroneousTokenKinds.includes(t.kind))
+if (errorTokens.length > 0) {
+    for (const t of errorTokens) {
         console.error(prettySourceMessage(prettyLexerError(t), t.location.start, source))
     }
     process.exit(1)

diff --git a/src/lexer/lexer.spec.ts b/src/lexer/lexer.spec.ts
@@ -73,9 +73,9 @@ let main = (): Unit {
     })
 
     it('tokenize string literal', () => {
-        expect(tokenize(`"string 123 \n ok"`)).toEqual([
-            { kind: 'string', value: `"string 123 \n ok"`, location: { start: 0, end: 16 } },
-            { kind: 'eof', value: '', location: { start: 17, end: 17 } }
+        expect(tokenize(`"string 123 ok"`)).toEqual([
+            { kind: 'string', value: `"string 123 ok"`, location: { start: 0, end: 14 } },
+            { kind: 'eof', value: '', location: { start: 15, end: 15 } }
         ])
     })
 

diff --git a/src/lexer/lexer.ts b/src/lexer/lexer.ts
@@ -51,10 +51,14 @@ export const lexerTokenKinds = <const>[
 
     // special
     'unknown',
+    'unterminated-string',
+    'unterminated-char',
     'eof'
 ]
 export type TokenKind = typeof lexerTokenKinds[number]
 
+export const erroneousTokenKinds: TokenKind[] = ['unknown', 'unterminated-char', 'unterminated-string']
+
 export interface ParseToken {
     kind: TokenKind
     value: string
@@ -266,6 +270,11 @@ const parseCharLiteral = (chars: string[], tokens: ParseToken[], pos: { pos: num
         pos.pos++
         const charLiteral: string[] = []
         while (chars[pos.pos] !== quote) {
+            if (isNewline(chars[pos.pos]) || pos.pos === chars.length) {
+                pos.pos++
+                tokens.push(createToken('unterminated-char', quote + charLiteral.join(''), pos, start))
+                return true
+            }
             charLiteral.push(chars[pos.pos])
             pos.pos++
         }
@@ -292,8 +301,10 @@ const parseStringLiteral = (chars: string[], tokens: ParseToken[], pos: { pos: n
         pos.pos++
         const stringLiteral: string[] = []
         while (chars[pos.pos] !== quote) {
-            if (chars.length === pos.pos) {
-                throw Error(`no matching \`${quote}\``)
+            if (isNewline(chars[pos.pos]) || pos.pos === chars.length) {
+                pos.pos++
+                tokens.push(createToken('unterminated-string', quote + stringLiteral.join(''), pos, start))
+                return true
             }
             stringLiteral.push(chars[pos.pos])
             pos.pos++