From 657a38df78b75e3a3ce82a2bf12281ba1bcbe915 Mon Sep 17 00:00:00 2001
From: ivanjermakov <ivanjermakov1@gmail.com>
Date: Sun, 11 Jun 2023 13:36:03 +0200
Subject: [PATCH] Lexer: floats regex fix; tests

---
 src/lexer/lexer.spec.ts | 23 +++++++++++++++++++++--
 src/lexer/lexer.ts      |  4 ++--
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/lexer/lexer.spec.ts b/src/lexer/lexer.spec.ts
index 38e6aecb..019ab02c 100644
--- a/src/lexer/lexer.spec.ts
+++ b/src/lexer/lexer.spec.ts
@@ -32,7 +32,7 @@ let main = (): Unit {
     })
 
     describe('tokenize int', () => {
-        it('tokenize int simple', () => {
+        it('simple', () => {
             expect(tokenize('14 2 0')).toEqual([
                 { kind: 'int', value: '14', location: { start: 0, end: 1 } },
                 { kind: 'int', value: '2', location: { start: 3, end: 3 } },
@@ -43,7 +43,7 @@ let main = (): Unit {
     })
 
     describe('tokenize float', () => {
-        it('tokenize float simple', () => {
+        it('simple', () => {
             expect(tokenize('14.0 2.0 0.0')).toEqual([
                 { kind: 'float', value: '14.0', location: { start: 0, end: 3 } },
                 { kind: 'float', value: '2.0', location: { start: 5, end: 7 } },
@@ -51,6 +51,25 @@ let main = (): Unit {
                 { kind: 'eof', value: '', location: { start: 12, end: 12 } }
             ])
         })
+
+        it('shorthand', () => {
+            expect(tokenize('14. .0 0. .11')).toEqual([
+                { kind: 'float', value: '14.', location: { start: 0, end: 2 } },
+                { kind: 'float', value: '.0', location: { start: 4, end: 5 } },
+                { kind: 'float', value: '0.', location: { start: 7, end: 8 } },
+                { kind: 'float', value: '.11', location: { start: 10, end: 12 } },
+                { kind: 'eof', value: '', location: { start: 13, end: 13 } }
+            ])
+        })
+
+        it('scientific', () => {
+            expect(tokenize('1e5 0e2 123.54e-1034')).toEqual([
+                { kind: 'float', value: '1e5', location: { start: 0, end: 2 } },
+                { kind: 'float', value: '0e2', location: { start: 4, end: 6 } },
+                { kind: 'float', value: '123.54e-1034', location: { start: 8, end: 19 } },
+                { kind: 'eof', value: '', location: { start: 20, end: 20 } }
+            ])
+        })
     })
 
     it('tokenize string literal', () => {
diff --git a/src/lexer/lexer.ts b/src/lexer/lexer.ts
index 50162ebc..1afe9a04 100644
--- a/src/lexer/lexer.ts
+++ b/src/lexer/lexer.ts
@@ -99,7 +99,7 @@ export const constTokenKindMap: Map<TokenKind, string> = new Map([
 ])
 
 const intRegex = /^\d+/
-const floatRegex = /^((\d+\.\d*)|(\d*\.\d+)|(\d+e[+-]?\d+))/
+const floatRegex = /^((\d+(\.\d*)?e[+-]?\d+)|(\d+\.\d*)|(\d*\.\d+))/
 
 /**
  * Independent tokens are automatically advanced by parser by default
@@ -132,7 +132,7 @@ export const tokenize = (code: String): ParseToken[] => {
             continue
         }
 
-        const fns = [parseComment, parseNewline, parseConstToken, parseIdentifier, parseFloat, parseInt,
+        const fns = [parseFloat, parseInt, parseComment, parseNewline, parseConstToken, parseIdentifier,
             parseCharLiteral, parseStringLiteral]
 
         let parsed = false