Skip to content

Commit 25487cc

Browse files
efekosefekos
committedApr 5, 2024
add positions to tokens
1 parent 12ec2d4 commit 25487cc

File tree

2 files changed

+40
-28
lines changed

2 files changed

+40
-28
lines changed
 

‎src/compiler/lexer.ts

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { Token, TokenType } from './types.js';
22
import { log } from '../log.js';
3+
import chalk from 'chalk';
34

45
const keywords: Record<string, TokenType> = {
56
operator: TokenType.OperatorKeyword,
@@ -64,49 +65,54 @@ function isInt(src: string) {
6465
export function tokenizeSyx(source: string, watchMode: boolean): Token[] {
6566
const tokens: Token[] = [];
6667
const src = source.split('');
68+
let curPos = -1;
6769

6870
while (src.length > 0) {
6971
if (!isSkippable(src[0])) log.debug(`Parsing token: '${src[0]}'`);
70-
if (src[0] === '(') tokens.push({ type: TokenType.OpenParen, value: src.shift() });
71-
else if (src[0] === ')') tokens.push({ type: TokenType.CloseParen, value: src.shift() });
72-
else if (src[0] === '{') tokens.push({ type: TokenType.OpenBrace, value: src.shift() });
73-
else if (src[0] === '}') tokens.push({ type: TokenType.CloseBrace, value: src.shift() });
74-
else if (src[0] === '[') tokens.push({ type: TokenType.OpenSquare, value: src.shift() });
75-
else if (src[0] === ']') tokens.push({ type: TokenType.CloseSquare, value: src.shift() });
76-
else if (src[0] === ',') tokens.push({ type: TokenType.Comma, value: src.shift() });
77-
else if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift() });
78-
else if (src[0] === '<') tokens.push({ type: TokenType.OpenDiamond, value: src.shift() });
79-
else if (src[0] === '>') tokens.push({ type: TokenType.CloseDiamond, value: src.shift() });
80-
else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift() });
81-
else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift() });
82-
else if (src[0] === '|') tokens.push({ type: TokenType.VarSeperator, value: src.shift() });
72+
if (src[0] === '(') tokens.push({ type: TokenType.OpenParen, value: src.shift(), pos:++curPos, end:curPos });
73+
else if (src[0] === ')') tokens.push({ type: TokenType.CloseParen, value: src.shift(), pos:++curPos, end: curPos });
74+
else if (src[0] === '{') tokens.push({ type: TokenType.OpenBrace, value: src.shift() ,pos:++curPos,end:curPos});
75+
else if (src[0] === '}') tokens.push({ type: TokenType.CloseBrace, value: src.shift() ,pos:++curPos,end:curPos});
76+
else if (src[0] === '[') tokens.push({ type: TokenType.OpenSquare, value: src.shift() ,pos:++curPos,end:curPos});
77+
else if (src[0] === ']') tokens.push({ type: TokenType.CloseSquare, value: src.shift() ,pos:++curPos,end:curPos});
78+
else if (src[0] === ',') tokens.push({ type: TokenType.Comma, value: src.shift() ,pos:++curPos,end:curPos});
79+
else if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift() ,pos:++curPos,end:curPos});
80+
else if (src[0] === '<') tokens.push({ type: TokenType.OpenDiamond, value: src.shift() ,pos:++curPos,end:curPos});
81+
else if (src[0] === '>') tokens.push({ type: TokenType.CloseDiamond, value: src.shift() ,pos:++curPos,end:curPos});
82+
else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift() ,pos:++curPos,end:curPos});
83+
else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift() ,pos:++curPos,end:curPos});
84+
else if (src[0] === '|') tokens.push({ type: TokenType.VarSeperator, value: src.shift() ,pos:++curPos,end:curPos});
8385
else if (src[0] === '+' && chars.includes(src[1])) {
84-
if (src[1] === 's') tokens.push({ type: TokenType.WhitespaceIdentifier, value: '+s' });
85-
else (watchMode ? log.thrower : log.exit).error(`Unexpected identifier: '${src[1]}'`);
86+
if (src[1] === 's') tokens.push({ type: TokenType.WhitespaceIdentifier, value: '+s' ,pos:++curPos,end:curPos+1});
87+
else (watchMode ? log.thrower : log.exit).error(`${chalk.gray(curPos)} Unexpected identifier: '${src[1]}'`);
8688
src.shift(); src.shift();
8789
} else if (isInt(src[0])) {
8890
log.debug('Found int number');
8991
let ident = '';
92+
const startPos = ++curPos;
9093
while (src.length > 0 && isInt(src[0])) {
9194
ident += src.shift();
95+
curPos++;
9296
}
9397

94-
tokens.push({ type: TokenType.IntNumber, value: ident });
98+
tokens.push({ type: TokenType.IntNumber, value: ident, pos:startPos,end: curPos });
9599
} else if (isAlphabetic(src[0])) {
96100
log.debug('Found identifier');
97101
let ident = '';
102+
const startPos = ++curPos;
98103
while (src.length > 0 && isAlphabetic(src[0])) {
99104
ident += src.shift();
105+
curPos++;
100106
}
101107

102108
const reserved = keywords[ident];
103109
if (reserved !== undefined) log.debug(`Found keyword: '${reserved}'`);
104-
tokens.push({ type: reserved ?? TokenType.Identifier, value: ident });
105-
} else if (isSkippable(src[0])) { log.debug('Found skippable char'); src.shift(); }
106-
else tokens.push({ type: TokenType.Raw, value: src.shift() });
110+
tokens.push({ type: reserved ?? TokenType.Identifier, value: ident,pos:startPos,end:curPos });
111+
} else if (isSkippable(src[0])) { log.debug('Found skippable char'); src.shift(); curPos++; }
112+
else tokens.push({ type: TokenType.Raw, value: src.shift(),pos:++curPos,end:curPos });
107113
}
108114

109-
tokens.push({ type: TokenType.EndOfFile, value: 'EOF' });
115+
tokens.push({ type: TokenType.EndOfFile, value: 'EOF',pos:source.length,end:source.length });
110116
return tokens;
111117
}
112118

@@ -121,27 +127,31 @@ export function tokenizeSyx(source: string, watchMode: boolean): Token[] {
121127
export function tokenizeSys(source: string): Token[] {
122128
const src = source.split('');
123129
const tokens: Token[] = [];
130+
131+
let curPos = -1;
124132

125133
while (src.length > 0 && `${src[0]}${src[1]}${src[2]}` !== ':::') {
126134
if (!isSkippable(src[0])) log.debug(`Parsing tokenmm: '${src[0]}'`);
127-
if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift() });
128-
else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift() });
129-
else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift() });
135+
if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift(), pos:++curPos, end:curPos });
136+
else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift(), pos:++curPos, end:curPos });
137+
else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift(), pos:++curPos, end:curPos });
130138
else if (isAlphabetic(src[0])) {
131139
log.debug('Found identifier');
132140
let ident = '';
141+
const startPost = ++curPos;
133142
while (src.length > 0 && isAlphabetic(src[0])) {
134143
ident += src.shift();
144+
curPos++;
135145
}
136146

137147
const reserved = keywords[ident];
138148
if (reserved !== undefined) log.debug(`Found keyword: '${reserved}'`);
139-
tokens.push({ type: reserved ?? TokenType.Identifier, value: ident });
140-
} else if (isSkippable(src[0])) { log.debug('Found skippable char'); src.shift(); }
141-
else tokens.push({ type: TokenType.Raw, value: src.shift() });
149+
tokens.push({ type: reserved ?? TokenType.Identifier, value: ident, pos:startPost, end:curPos });
150+
} else if (isSkippable(src[0])) { log.debug('Found skippable char'); src.shift(); curPos++; }
151+
else tokens.push({ type: TokenType.Raw, value: src.shift(), pos:++curPos,end:curPos });
142152

143153
}
144154

145-
tokens.push({ type: TokenType.EndOfFile, value: 'eof' });
155+
tokens.push({ type: TokenType.EndOfFile, value: 'eof', pos:++curPos,end:curPos });
146156
return tokens;
147157
}

‎src/compiler/types.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,15 @@ export enum TokenType {
3939

4040
/**
4141
* Base token interface.
42-
* @version 1.0.0
42+
* @version 1.0.1
4343
* @since 0.0.1-alpha
4444
* @author efekos
4545
*/
4646
export interface Token {
4747
type: TokenType;
4848
value: string;
49+
pos:number;
50+
end:number;
4951
}
5052

5153
/**

0 commit comments

Comments
 (0)
Please sign in to comment.