diff --git a/lib/coffeescript/coffeescript.js b/lib/coffeescript/coffeescript.js index ea938c2980..81e443608c 100644 --- a/lib/coffeescript/coffeescript.js +++ b/lib/coffeescript/coffeescript.js @@ -4,10 +4,10 @@ // on Node.js/V8, or to run CoffeeScript directly in the browser. This module // contains the main entry functions for tokenizing, parsing, and compiling // source CoffeeScript into JavaScript. - var FILE_EXTENSIONS, Lexer, SourceMap, base64encode, checkShebangLine, compile, formatSourcePosition, getSourceMap, helpers, lexer, packageJson, parser, registerCompiled, sourceMaps, sources, withPrettyErrors, + var FILE_EXTENSIONS, LINEBREAK, Lexer, SourceMap, base64encode, checkShebangLine, compile, formatSourcePosition, getSourceMap, helpers, lexer, packageJson, parser, registerCompiled, sourceMaps, sources, withPrettyErrors, indexOf = [].indexOf; - ({Lexer} = require('./lexer')); + ({Lexer, LINEBREAK} = require('./lexer')); ({parser} = require('./parser')); @@ -147,7 +147,7 @@ // `clean` function in the lexer). if (options.ast) { nodes.allCommentTokens = helpers.extractAllCommentTokens(tokens); - sourceCodeNumberOfLines = (code.match(/\r?\n/g) || '').length + 1; + sourceCodeNumberOfLines = (code.match(LINEBREAK) || []).length + 1; sourceCodeLastLine = /.*$/.exec(code)[0]; ast = nodes.ast(options); range = [0, code.length]; diff --git a/lib/coffeescript/lexer.js b/lib/coffeescript/lexer.js index 6f6c9e3a5d..4077b7d537 100644 --- a/lib/coffeescript/lexer.js +++ b/lib/coffeescript/lexer.js @@ -10,7 +10,7 @@ // where locationData is {first_line, first_column, last_line, last_column, last_line_exclusive, last_column_exclusive}, which is a // format that can be fed directly into [Jison](https://github.com/zaach/jison). These // are read by jison in the `parser.lexer` function defined in coffeescript.coffee. - var BOM, BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARABLE_LEFT_SIDE, COMPARE, COMPOUND_ASSIGN, HERECOMMENT_ILLEGAL, HEREDOC_DOUBLE, HEREDOC_INDENT, HEREDOC_SINGLE, HEREGEX, HEREGEX_COMMENT, HERE_JSTOKEN, IDENTIFIER, INDENTABLE_CLOSERS, INDEXABLE, INSIDE_JSX, INVERSES, JSTOKEN, JSX_ATTRIBUTE, JSX_FRAGMENT_IDENTIFIER, JSX_IDENTIFIER, JSX_IDENTIFIER_PART, JSX_INTERPOLATION, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, Lexer, MATH, MULTI_DENT, NOT_REGEX, NUMBER, OPERATOR, POSSIBLY_DIVISION, REGEX, REGEX_FLAGS, REGEX_ILLEGAL, REGEX_INVALID_ESCAPE, RELATION, RESERVED, Rewriter, SHIFT, STRICT_PROSCRIBED, STRING_DOUBLE, STRING_INVALID_ESCAPE, STRING_SINGLE, STRING_START, TRAILING_SPACES, UNARY, UNARY_MATH, UNFINISHED, VALID_FLAGS, WHITESPACE, addTokenData, attachCommentsToNode, compact, count, flatten, invertLiterate, isForFrom, isUnassignable, key, locationDataToString, merge, parseNumber, repeat, replaceUnicodeCodePointEscapes, starts, throwSyntaxError, + var BOM, BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARABLE_LEFT_SIDE, COMPARE, COMPOUND_ASSIGN, HERECOMMENT_ILLEGAL, HEREDOC_DOUBLE, HEREDOC_INDENT, HEREDOC_SINGLE, HEREGEX, HEREGEX_COMMENT, HERE_JSTOKEN, IDENTIFIER, INDENTABLE_CLOSERS, INDEXABLE, INSIDE_JSX, INVERSES, JSTOKEN, JSX_ATTRIBUTE, JSX_FRAGMENT_IDENTIFIER, JSX_IDENTIFIER, JSX_IDENTIFIER_PART, JSX_INTERPOLATION, JS_KEYWORDS, LINEBREAK, LINE_BREAK, LINE_CONTINUER, Lexer, MATH, MULTI_DENT, NOT_REGEX, NUMBER, OPERATOR, POSSIBLY_DIVISION, REGEX, REGEX_FLAGS, REGEX_ILLEGAL, REGEX_INVALID_ESCAPE, RELATION, RESERVED, Rewriter, SHIFT, STRICT_PROSCRIBED, STRING_DOUBLE, STRING_INVALID_ESCAPE, STRING_SINGLE, STRING_START, TRAILING_SPACES, UNARY, UNARY_MATH, UNFINISHED, VALID_FLAGS, WHITESPACE, addTokenData, attachCommentsToNode, compact, count, flatten, invertLiterate, isForFrom, isUnassignable, key, locationDataToString, merge, parseNumber, repeat, replaceUnicodeCodePointEscapes, starts, throwSyntaxError, indexOf = [].indexOf, slice = [].slice; @@ -1532,10 +1532,10 @@ } else { string = this.chunk.slice(0, +(offset - 1) + 1 || 9e9); } - lineCount = count(string, '\n'); + lineCount = (string.match(LINEBREAK) || []).length; column = this.chunkColumn; if (lineCount > 0) { - ref = string.split('\n'), [lastLine] = slice.call(ref, -1); + ref = string.split(LINEBREAK), [lastLine] = slice.call(ref, -1); column = lastLine.length; previousLinesCompensation = this.getLocationDataCompensation(this.chunkOffset, this.chunkOffset + offset - column); if (previousLinesCompensation < 0) { @@ -1808,6 +1808,8 @@ WHITESPACE = /^[^\n\S]+/; + exports.LINEBREAK = LINEBREAK = /\r\n|[\r\n\u2028\u2029]/ug; + COMMENT = /^(\s*)###([^#][\s\S]*?)(?:###([^\n\S]*)|###$)|^((?:\s*#(?!##[^#]).*)+)/; CODE = /^[-=]>/; diff --git a/src/coffeescript.coffee b/src/coffeescript.coffee index 6cc9e07b25..bc7fc3eeb3 100644 --- a/src/coffeescript.coffee +++ b/src/coffeescript.coffee @@ -3,7 +3,7 @@ # contains the main entry functions for tokenizing, parsing, and compiling # source CoffeeScript into JavaScript. -{Lexer} = require './lexer' +{Lexer, LINEBREAK} = require './lexer' {parser} = require './parser' helpers = require './helpers' SourceMap = require './sourcemap' @@ -114,7 +114,7 @@ exports.compile = compile = withPrettyErrors (code, options = {}) -> # `clean` function in the lexer). if options.ast nodes.allCommentTokens = helpers.extractAllCommentTokens tokens - sourceCodeNumberOfLines = (code.match(/\r?\n/g) or '').length + 1 + sourceCodeNumberOfLines = (code.match(LINEBREAK) or []).length + 1 sourceCodeLastLine = /.*$/.exec(code)[0] # `.*` matches all but line break characters. ast = nodes.ast options range = [0, code.length] diff --git a/src/lexer.coffee b/src/lexer.coffee index 8c9f68ceb6..d5420b8d1a 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -1056,11 +1056,11 @@ exports.Lexer = class Lexer else string = @chunk[..offset-1] - lineCount = count string, '\n' + lineCount = (string.match(LINEBREAK) or []).length column = @chunkColumn if lineCount > 0 - [..., lastLine] = string.split '\n' + [..., lastLine] = string.split LINEBREAK column = lastLine.length previousLinesCompensation = @getLocationDataCompensation @chunkOffset, @chunkOffset + offset - column # Don't recompensate for initially inserted newline. @@ -1318,6 +1318,8 @@ OPERATOR = /// ^ ( WHITESPACE = /^[^\n\S]+/ +exports.LINEBREAK = LINEBREAK = /\r\n|[\r\n\u2028\u2029]/ug + COMMENT = /^(\s*)###([^#][\s\S]*?)(?:###([^\n\S]*)|###$)|^((?:\s*#(?!##[^#]).*)+)/ CODE = /^[-=]>/