Merge pull request stadelmanma#49 from stadelmanma/use-external-scanner-for-number-literals

Use external scanner for number literals

Author: stadelmanma

grammar.js

@@ -51,7 +51,10 @@ module.exports = grammar({
   name: 'fortran',
 
   externals: $ => [
-    $._line_continuation
+    $._line_continuation,
+    $._integer_literal,
+    $._float_literal,
+    $._boz_literal
   ],
 
   extras: $ => [
@@ -522,9 +525,9 @@ module.exports = grammar({
       $.read_statement
     ),
 
-    statement_label: $ => /\d+/,
+    statement_label: $ => prec(1, alias($._integer_literal, 'statement_label')),
 
-    _statement_label_reference: $ => alias($.statement_label, $.statement_label_reference),
+    statement_label_reference: $ => alias($.statement_label, 'statement_label_reference'),
 
     assignment_statement: $ => prec.right(PREC.ASSIGNMENT, seq(
       $._expression,
@@ -827,7 +830,7 @@ module.exports = grammar({
     )),
 
     format_identifier: $ => choice(
-      $._statement_label_reference,
+      $.statement_label_reference,
       $._io_expressions
     ),
 
@@ -1010,27 +1013,6 @@ module.exports = grammar({
 
     array_literal: $ => seq('(/', commaSep1($._expression), '/)'),
 
-    number_literal: $ => token(
-      choice(
-        // integer, real with and without exponential notation
-        /(((\d*\.)?\d+)|(\d+(\.\d*)?))([eEdD][-+]?\d+)?(_[a-zA-Z_]+)?/,
-        // binary literal
-        /[bB]'[01]+'/,
-        /'[01]+'[bB]/,
-        /[bB]"[01]+"/,
-        /"[01]+"[bB]/,
-        // octal literal
-        /[oO]'[0-8]+'/,
-        /'[0-8]+'[oO]/,
-        /[oO]"[0-8]+"/,
-        /"[0-8]+"[oO]/,
-        // hexcadecimal
-        /[zZ]'[0-9a-fA-F]+'/,
-        /'[0-9a-fA-F]+'[zZ]/,
-        /[zZ]"[0-9a-fA-F]+"/,
-        /"[0-9a-fA-F]+"[zZ]/
-      )),
-
     complex_literal: $ => seq(
       '(',
       choice($.number_literal, $.identifier),
@@ -1039,6 +1021,12 @@ module.exports = grammar({
       ')'
     ),
 
+    number_literal: $ => choice(
+      $._integer_literal,
+      $._float_literal,
+      $._boz_literal
+    ),
+
     string_literal: $ => choice(
       $._double_quoted_string,
       $._single_quoted_string

src/grammar.json

@@ -2594,16 +2594,25 @@
       ]
     },
     "statement_label": {
-      "type": "PATTERN",
-      "value": "\\d+"
+      "type": "PREC",
+      "value": 1,
+      "content": {
+        "type": "ALIAS",
+        "content": {
+          "type": "SYMBOL",
+          "name": "_integer_literal"
+        },
+        "named": false,
+        "value": "statement_label"
+      }
     },
-    "_statement_label_reference": {
+    "statement_label_reference": {
       "type": "ALIAS",
       "content": {
         "type": "SYMBOL",
         "name": "statement_label"
       },
-      "named": true,
+      "named": false,
       "value": "statement_label_reference"
     },
     "assignment_statement": {
@@ -3801,7 +3810,7 @@
     },
     "edit_descriptor": {
       "type": "PATTERN",
-      "value": "[a-zA-Z0-9\\/:.*]+"
+      "value": "[a-zA-Z0-9/:.*]+"
     },
     "read_statement": {
       "type": "CHOICE",
@@ -4323,7 +4332,7 @@
       "members": [
         {
           "type": "SYMBOL",
-          "name": "_statement_label_reference"
+          "name": "statement_label_reference"
         },
         {
           "type": "SYMBOL",
@@ -5615,66 +5624,6 @@
         }
       ]
     },
-    "number_literal": {
-      "type": "TOKEN",
-      "content": {
-        "type": "CHOICE",
-        "members": [
-          {
-            "type": "PATTERN",
-            "value": "(((\\d*\\.)?\\d+)|(\\d+(\\.\\d*)?))([eEdD][-+]?\\d+)?(_[a-zA-Z_]+)?"
-          },
-          {
-            "type": "PATTERN",
-            "value": "[bB]'[01]+'"
-          },
-          {
-            "type": "PATTERN",
-            "value": "'[01]+'[bB]"
-          },
-          {
-            "type": "PATTERN",
-            "value": "[bB]\"[01]+\""
-          },
-          {
-            "type": "PATTERN",
-            "value": "\"[01]+\"[bB]"
-          },
-          {
-            "type": "PATTERN",
-            "value": "[oO]'[0-8]+'"
-          },
-          {
-            "type": "PATTERN",
-            "value": "'[0-8]+'[oO]"
-          },
-          {
-            "type": "PATTERN",
-            "value": "[oO]\"[0-8]+\""
-          },
-          {
-            "type": "PATTERN",
-            "value": "\"[0-8]+\"[oO]"
-          },
-          {
-            "type": "PATTERN",
-            "value": "[zZ]'[0-9a-fA-F]+'"
-          },
-          {
-            "type": "PATTERN",
-            "value": "'[0-9a-fA-F]+'[zZ]"
-          },
-          {
-            "type": "PATTERN",
-            "value": "[zZ]\"[0-9a-fA-F]+\""
-          },
-          {
-            "type": "PATTERN",
-            "value": "\"[0-9a-fA-F]+\"[zZ]"
-          }
-        ]
-      }
-    },
     "complex_literal": {
       "type": "SEQ",
       "members": [
@@ -5718,6 +5667,23 @@
         }
       ]
     },
+    "number_literal": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "_integer_literal"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "_float_literal"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "_boz_literal"
+        }
+      ]
+    },
     "string_literal": {
       "type": "CHOICE",
       "members": [
@@ -5872,6 +5838,18 @@
     {
       "type": "SYMBOL",
       "name": "_line_continuation"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_integer_literal"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_float_literal"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_boz_literal"
     }
   ],
   "inline": [