Merge branch 'fix-handle-function-token-matchers' of https://github.com/airportyh/nearley into airportyh-fix-handle-function-token-matchers

Author: kach

examples/token-2.ne

@@ -0,0 +1,6 @@
+@{%
+const tokenPrint = { literal: "print" };
+const tokenNumber = { test: x => Number.isInteger(x) };
+%}
+
+main -> %tokenPrint %tokenNumber ";;"

lib/nearley.js

@@ -16,15 +16,11 @@
     Rule.highestId = 0;
 
     Rule.prototype.toString = function(withCursorAt) {
-        function stringifySymbolSequence (e) {
-            return e.literal ? JSON.stringify(e.literal) :
-                   e.type ? '%' + e.type : e.toString();
-        }
         var symbolSequence = (typeof withCursorAt === "undefined")
-                             ? this.symbols.map(stringifySymbolSequence).join(' ')
-                             : (   this.symbols.slice(0, withCursorAt).map(stringifySymbolSequence).join(' ')
+                             ? this.symbols.map(getSymbolShortDisplay).join(' ')
+                             : (   this.symbols.slice(0, withCursorAt).map(getSymbolShortDisplay).join(' ')
                                  + " ● "
-                                 + this.symbols.slice(withCursorAt).map(stringifySymbolSequence).join(' ')     );
+                                 + this.symbols.slice(withCursorAt).map(getSymbolShortDisplay).join(' ')     );
         return this.name + " → " + symbolSequence;
     }
 
@@ -391,18 +387,7 @@
     };
 
     Parser.prototype.getSymbolDisplay = function(symbol) {
-        var type = typeof symbol;
-        if (type === "string") {
-            return symbol;
-        } else if (type === "object" && symbol.literal) {
-            return JSON.stringify(symbol.literal);
-        } else if (type === "object" && symbol instanceof RegExp) {
-            return 'character matching ' + symbol;
-        } else if (type === "object" && symbol.type) {
-            return symbol.type + ' token';
-        } else {
-            throw new Error('Unknown symbol type: ' + symbol);
-        }
+        return getSymbolLongDisplay(symbol);
     };
 
     /*
@@ -478,6 +463,44 @@
         return considerations.map(function(c) {return c.data; });
     };
 
+    function getSymbolLongDisplay(symbol) {
+        var type = typeof symbol;
+        if (type === "string") {
+            return symbol;
+        } else if (type === "object") {
+            if (symbol.literal) {
+                return JSON.stringify(symbol.literal);
+            } else if (symbol instanceof RegExp) {
+                return 'character matching ' + symbol;
+            } else if (symbol.type) {
+                return symbol.type + ' token';
+            } else if (symbol.test) {
+                return 'token matching ' + String(symbol.test);
+            } else {
+                throw new Error('Unknown symbol type: ' + symbol);
+            }
+        }
+    }
+
+    function getSymbolShortDisplay(symbol) {
+        var type = typeof symbol;
+        if (type === "string") {
+            return symbol;
+        } else if (type === "object") {
+            if (symbol.literal) {
+                return JSON.stringify(symbol.literal);
+            } else if (symbol instanceof RegExp) {
+                return symbol.toString();
+            } else if (symbol.type) {
+                return '%' + symbol.type;
+            } else if (symbol.test) {
+                return '<' + String(symbol.test) + '>';
+            } else {
+                throw new Error('Unknown symbol type: ' + symbol);
+            }
+        }
+    }
+
     return {
         Parser: Parser,
         Grammar: Grammar,

test/parser.test.js

@@ -183,6 +183,11 @@ describe('Parser: examples', () => {
         expect(parse(tokc, [123, 456, " ", 789])).toEqual([ [123, [ [ 456, " ", 789 ] ]] ]);
     });
 
+    it('tokens 2', () => {
+        var tokc = compile(read("examples/token-2.ne"));
+        expect(() => parse(tokc, ["print", "blah", 12, ";", ";"])).toThrow(/A token matching x \=\> Number\.isInteger\(x\)/);
+    })
+
     const json = compile(read("examples/json.ne"));
     it('json', () => {
         const test1 = '{ "a" : true, "b" : "䕧⡱a\\\\\\"b\\u4567\\u2871䕧⡱\\t\\r\\f\\b\\n", "c" : null, "d" : [null, true, false, null] }\n'