Manual parser & some rules

nois-lang · Jun 7, 2023 · ddea266 · ddea266
1 parent 2fe56f6
commit ddea266
Show file tree

Hide file tree

Showing 5 changed files with 338 additions and 38 deletions.
diff --git a/nois.bnf b/nois.bnf
@@ -20,7 +20,7 @@ statement           ::= variable-def | type-def | return-stmt | expr
                     | CHAR
                     | NUMBER
                     | IDENTIFIER
-                    | type
+                    | type-expr
                     ;
     infix-op        ::= add-op | sub-op | mul-op | div-op | exp-op | mod-op | access-op | eq-op | ne-op
                     | ge-op | le-op | gt-op | lt-op | and-op | or-op;
@@ -53,18 +53,18 @@ statement           ::= variable-def | type-def | return-stmt | expr
                     ;
         args        ::= expr (COMMA expr)* COMMA?
                     ;
-    fn-expr         ::= O-PAREN params C-PAREN COLON type block
+    fn-expr         ::= PIPE params PIPE COLON type-expr block
                     ;
       params        ::= param (COMMA param)* COMMA?
                     ;
-        param       ::= IDENTIFIER COLON type
+        param       ::= IDENTIFIER COLON type-expr
                     ;
 block               ::= O-BRACE statement* C-BRACE
                     | O-BRACE C-BRACE
                     ;
-type                ::= IDENTIFIER (O-ANGLE type-params C-ANGLE)?
+type-expr           ::= IDENTIFIER (O-ANGLE type-params C-ANGLE)?
                     ;
-  type-params       ::= type (COMMA type)* COMMA?
+  type-params       ::= type-expr (COMMA type-expr)* COMMA?
                     ;
 if-expr             ::= IF-KEYWORD expr block (ELSE-KEYWORD block)?
                     ;
diff --git a/src/error.ts b/src/error.ts
@@ -1,15 +1,14 @@
-import { TokenKind } from './parser/parser'
 import { indexToLocation, LocationRange, prettyIndex, prettyLocation } from './location'
 import { Source } from './source'
-import { Token } from './lexer/lexer'
+import { ParseToken, TokenKind } from './lexer/lexer'
 
 export interface SyntaxErrorInfo {
     expected: TokenKind[],
     got: TokenKind,
     location: LocationRange
 }
 
-export const prettyLexerError = (token: Token): string => {
+export const prettyLexerError = (token: ParseToken): string => {
     return `lexer error: unknown token \`${token.value}\``
 }
 

diff --git a/src/lexer/lexer.ts b/src/lexer/lexer.ts
@@ -45,15 +45,15 @@ export const lexerTokenKinds = <const>[
     'unknown',
     'eof'
 ]
-export type LexerTokenKind = typeof lexerTokenKinds[number]
+export type TokenKind = typeof lexerTokenKinds[number]
 
-export interface Token {
-    kind: LexerTokenKind
+export interface ParseToken {
+    kind: TokenKind
     value: string
     location: LocationRange
 }
 
-export const constLexerTokenMap: Map<LexerTokenKind, string> = new Map([
+export const constTokenKindMap: Map<TokenKind, string> = new Map([
     ['type-keyword', 'type'],
     ['kind-keyword', 'kind'],
     ['if-keyword', 'if'],
@@ -91,12 +91,12 @@ export const isWhitespace = (char: string): boolean => char === ' ' || char ===
 
 export const isNewline = (char: string): boolean => char === '\n' || char === '\r'
 
-export const tokenize = (code: String): Token[] => {
+export const tokenize = (code: String): ParseToken[] => {
     const pos = { pos: 0 }
     const chars = code.split('')
-    const tokens: Token[] = []
+    const tokens: ParseToken[] = []
 
-    let unknownToken: Token | undefined = undefined
+    let unknownToken: ParseToken | undefined = undefined
 
     const flushUnknown = () => {
         if (unknownToken) {
@@ -152,9 +152,9 @@ export const tokenize = (code: String): Token[] => {
     return tokens
 }
 
-const parseConstToken = (chars: string[], tokens: Token[], pos: { pos: number }): boolean => {
+const parseConstToken = (chars: string[], tokens: ParseToken[], pos: { pos: number }): boolean => {
     let codeLeft = chars.slice(pos.pos).join('')
-    let pair = [...constLexerTokenMap.entries()].find(([, v]) => codeLeft.startsWith(v))
+    let pair = [...constTokenKindMap.entries()].find(([, v]) => codeLeft.startsWith(v))
     if (pair) {
         const [kind, value] = pair
         const start = pos.pos
@@ -165,7 +165,7 @@ const parseConstToken = (chars: string[], tokens: Token[], pos: { pos: number })
     return false
 }
 
-const parseIdentifier = (chars: string[], tokens: Token[], pos: { pos: number }): boolean => {
+const parseIdentifier = (chars: string[], tokens: ParseToken[], pos: { pos: number }): boolean => {
     if (isAlpha(chars[pos.pos])) {
         const start = pos.pos
         const identifier: string[] = []
@@ -189,7 +189,7 @@ const parseIdentifier = (chars: string[], tokens: Token[], pos: { pos: number })
  * @param tokens
  * @param pos
  */
-const parseNumberLiteral = (chars: string[], tokens: Token[], pos: { pos: number }): boolean => {
+const parseNumberLiteral = (chars: string[], tokens: ParseToken[], pos: { pos: number }): boolean => {
     if (isNumeric(chars[pos.pos])) {
         const start = pos.pos
         const number: string[] = []
@@ -211,7 +211,7 @@ const parseNumberLiteral = (chars: string[], tokens: Token[], pos: { pos: number
  * @param tokens
  * @param pos
  */
-const parseCharLiteral = (chars: string[], tokens: Token[], pos: { pos: number }): boolean => {
+const parseCharLiteral = (chars: string[], tokens: ParseToken[], pos: { pos: number }): boolean => {
     const quote = `'`
     if (chars[pos.pos] === quote) {
         const start = pos.pos
@@ -236,7 +236,7 @@ const parseCharLiteral = (chars: string[], tokens: Token[], pos: { pos: number }
  * @param tokens
  * @param pos
  */
-const parseStringLiteral = (chars: string[], tokens: Token[], pos: { pos: number }): boolean => {
+const parseStringLiteral = (chars: string[], tokens: ParseToken[], pos: { pos: number }): boolean => {
     const quote = '"'
     if (chars[pos.pos] === quote) {
         const start = pos.pos
@@ -258,10 +258,10 @@ const parseStringLiteral = (chars: string[], tokens: Token[], pos: { pos: number
 }
 
 const createToken = (
-    name: LexerTokenKind,
+    name: TokenKind,
     value: string, pos: { pos: number },
     start: number = pos.pos
-): Token => {
+): ParseToken => {
     return { kind: name, value, location: { start, end: pos.pos - 1 } }
 }
 

diff --git a/src/parser/parser.spec.ts b/src/parser/parser.spec.ts
@@ -1,2 +1,4 @@
 describe('parser', () => {
+    it('buildTree', () => {
+    })
 })