add positions to tokens

Author: efekos

src/compiler/lexer.ts

@@ -1,5 +1,6 @@
 import { Token, TokenType } from './types.js';
 import { log } from '../log.js';
+import chalk from 'chalk';
 
 const keywords: Record<string, TokenType> = {
     operator: TokenType.OperatorKeyword,
@@ -64,49 +65,54 @@ function isInt(src: string) {
 export function tokenizeSyx(source: string, watchMode: boolean): Token[] {
     const tokens: Token[] = [];
     const src = source.split('');
+    let curPos = -1;
 
     while (src.length > 0) {
         if (!isSkippable(src[0])) log.debug(`Parsing token: '${src[0]}'`);
-        if (src[0] === '(') tokens.push({ type: TokenType.OpenParen, value: src.shift() });
-        else if (src[0] === ')') tokens.push({ type: TokenType.CloseParen, value: src.shift() });
-        else if (src[0] === '{') tokens.push({ type: TokenType.OpenBrace, value: src.shift() });
-        else if (src[0] === '}') tokens.push({ type: TokenType.CloseBrace, value: src.shift() });
-        else if (src[0] === '[') tokens.push({ type: TokenType.OpenSquare, value: src.shift() });
-        else if (src[0] === ']') tokens.push({ type: TokenType.CloseSquare, value: src.shift() });
-        else if (src[0] === ',') tokens.push({ type: TokenType.Comma, value: src.shift() });
-        else if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift() });
-        else if (src[0] === '<') tokens.push({ type: TokenType.OpenDiamond, value: src.shift() });
-        else if (src[0] === '>') tokens.push({ type: TokenType.CloseDiamond, value: src.shift() });
-        else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift() });
-        else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift() });
-        else if (src[0] === '|') tokens.push({ type: TokenType.VarSeperator, value: src.shift() });
+        if (src[0] === '(') tokens.push({ type: TokenType.OpenParen, value: src.shift(), pos:++curPos, end:curPos });
+        else if (src[0] === ')') tokens.push({ type: TokenType.CloseParen, value: src.shift(), pos:++curPos, end: curPos });
+        else if (src[0] === '{') tokens.push({ type: TokenType.OpenBrace, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === '}') tokens.push({ type: TokenType.CloseBrace, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === '[') tokens.push({ type: TokenType.OpenSquare, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === ']') tokens.push({ type: TokenType.CloseSquare, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === ',') tokens.push({ type: TokenType.Comma, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === '<') tokens.push({ type: TokenType.OpenDiamond, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === '>') tokens.push({ type: TokenType.CloseDiamond, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift() ,pos:++curPos,end:curPos});
+        else if (src[0] === '|') tokens.push({ type: TokenType.VarSeperator, value: src.shift() ,pos:++curPos,end:curPos});
         else if (src[0] === '+' && chars.includes(src[1])) {
-            if (src[1] === 's') tokens.push({ type: TokenType.WhitespaceIdentifier, value: '+s' });
-            else (watchMode ? log.thrower : log.exit).error(`Unexpected identifier: '${src[1]}'`);
+            if (src[1] === 's') tokens.push({ type: TokenType.WhitespaceIdentifier, value: '+s' ,pos:++curPos,end:curPos+1});
+            else (watchMode ? log.thrower : log.exit).error(`${chalk.gray(curPos)} Unexpected identifier: '${src[1]}'`);
             src.shift(); src.shift();
         } else if (isInt(src[0])) {
             log.debug('Found int number');
             let ident = '';
+            const startPos = ++curPos;
             while (src.length > 0 && isInt(src[0])) {
                 ident += src.shift();
+                curPos++;
             }
 
-            tokens.push({ type: TokenType.IntNumber, value: ident });
+            tokens.push({ type: TokenType.IntNumber, value: ident, pos:startPos,end: curPos });
         } else if (isAlphabetic(src[0])) {
             log.debug('Found identifier');
             let ident = '';
+            const startPos = ++curPos;
             while (src.length > 0 && isAlphabetic(src[0])) {
                 ident += src.shift();
+                curPos++;
             }
 
             const reserved = keywords[ident];
             if (reserved !== undefined) log.debug(`Found keyword: '${reserved}'`);
-            tokens.push({ type: reserved ?? TokenType.Identifier, value: ident });
-        } else if (isSkippable(src[0])) { log.debug('Found skippable char'); src.shift(); }
-        else tokens.push({ type: TokenType.Raw, value: src.shift() });
+            tokens.push({ type: reserved ?? TokenType.Identifier, value: ident,pos:startPos,end:curPos });
+        } else if (isSkippable(src[0])) { log.debug('Found skippable char'); src.shift(); curPos++; }
+        else tokens.push({ type: TokenType.Raw, value: src.shift(),pos:++curPos,end:curPos });
     }
 
-    tokens.push({ type: TokenType.EndOfFile, value: 'EOF' });
+    tokens.push({ type: TokenType.EndOfFile, value: 'EOF',pos:source.length,end:source.length });
     return tokens;
 }
 
@@ -121,27 +127,31 @@ export function tokenizeSyx(source: string, watchMode: boolean): Token[] {
 export function tokenizeSys(source: string): Token[] {
     const src = source.split('');
     const tokens: Token[] = [];
+    
+    let curPos = -1;
 
     while (src.length > 0 && `${src[0]}${src[1]}${src[2]}` !== ':::') {
         if (!isSkippable(src[0])) log.debug(`Parsing tokenmm: '${src[0]}'`);
-        if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift() });
-        else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift() });
-        else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift() });
+        if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift(), pos:++curPos, end:curPos });
+        else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift(), pos:++curPos, end:curPos });
+        else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift(), pos:++curPos, end:curPos });
         else if (isAlphabetic(src[0])) {
             log.debug('Found identifier');
             let ident = '';
+            const startPost = ++curPos;
             while (src.length > 0 && isAlphabetic(src[0])) {
                 ident += src.shift();
+                curPos++;
             }
 
             const reserved = keywords[ident];
             if (reserved !== undefined) log.debug(`Found keyword: '${reserved}'`);
-            tokens.push({ type: reserved ?? TokenType.Identifier, value: ident });
-        } else if (isSkippable(src[0])) { log.debug('Found skippable char'); src.shift(); }
-        else tokens.push({ type: TokenType.Raw, value: src.shift() });
+            tokens.push({ type: reserved ?? TokenType.Identifier, value: ident, pos:startPost, end:curPos });
+        } else if (isSkippable(src[0])) { log.debug('Found skippable char'); src.shift(); curPos++; }
+        else tokens.push({ type: TokenType.Raw, value: src.shift(), pos:++curPos,end:curPos });
 
     }
 
-    tokens.push({ type: TokenType.EndOfFile, value: 'eof' });
+    tokens.push({ type: TokenType.EndOfFile, value: 'eof', pos:++curPos,end:curPos });
     return tokens;
 }

src/compiler/types.ts

@@ -39,13 +39,15 @@ export enum TokenType {
 
 /**
  * Base token interface.
- * @version 1.0.0
+ * @version 1.0.1
  * @since 0.0.1-alpha
  * @author efekos
  */
 export interface Token {
     type: TokenType;
     value: string;
+    pos:number;
+    end:number;
 }
 
 /**