Lexer/Parser: Fix break/continue/return parsing

There were a number of issues around parsing these keywords when
when the code doesn't contain semi-colons. For example, this valid
code fragment would fail to parse:

     function f (x) {
         if (x) return
         x.foo()
     }

The solution was to modify the lexer so that it inserts an AutoSemiToken
after break/continue/return if and only if the white space immediately
following those keywords contain a newline.

Closes: erikd#30
Closes: erikd#36

Author: erikd

runtests.hs

@@ -16,13 +16,12 @@ main = defaultMain
     , parserSuite
     -- ++AZ++temporary++ , commentSuite
     , commentPrintSuite
-    -- , pendingSuite
+    , pendingSuite
     ]
 
 pendingSuite :: Test
 pendingSuite = testGroup "Pending"
-    [ testCase "AutoSemi1"  (testProg "if (true) return\nfoo();"  "Right (JSSourceElementsTop [JSIf (JSExpression [JSLiteral \"true\"]) ([JSReturn [] JSLiteral \"\"]) ([]),JSExpression [JSIdentifier \"foo\",JSArguments []],JSLiteral \";\"])")
-    , testCase "AutoSemi2"  (testProg "if (true) break\nfoo();"   "Right")
+    [
     ]
 
 lexerSuite:: Test
@@ -331,6 +330,9 @@ parserSuite = testGroup "Parser"
      -- https://github.com/alanz/language-javascript/issues/14
     , testCase "issue14" (testProg "var z = x[i] / y;" "Right (JSSourceElementsTop [JSVariables JSLiteral \"var\" [JSVarDecl (JSIdentifier \"z\") [JSLiteral \"=\",JSExpressionBinary \"/\" [JSMemberSquare [JSIdentifier \"x\"] (JSExpression [JSIdentifier \"i\"])] [JSIdentifier \"y\"]]],JSLiteral \"\"])")
 
+    , testCase "AutoSemiBreak"     (testProg "if(true)break \nfoo();"      "Right (JSSourceElementsTop [JSIf (JSExpression [JSLiteral \"true\"]) ([JSBreak [] JSLiteral \"\"]) ([]),JSExpression [JSIdentifier \"foo\",JSArguments []],JSLiteral \";\",JSLiteral \"\"])")
+    , testCase "AutoSemiContinue"  (testProg "if(true)continue \nfoo();"   "Right (JSSourceElementsTop [JSIf (JSExpression [JSLiteral \"true\"]) ([JSContinue [] JSLiteral \"\"]) ([]),JSExpression [JSIdentifier \"foo\",JSArguments []],JSLiteral \";\",JSLiteral \"\"])")
+    , testCase "AutoSemiReturn"    (testProg "if(true)break \nfoo();"      "Right (JSSourceElementsTop [JSIf (JSExpression [JSLiteral \"true\"]) ([JSBreak [] JSLiteral \"\"]) ([]),JSExpression [JSIdentifier \"foo\",JSArguments []],JSLiteral \";\",JSLiteral \"\"])")
     ]
 
 caseHelloWorld :: Assertion

src/Language/JavaScript/Parser/Grammar5.y

@@ -71,6 +71,7 @@ import qualified Language.JavaScript.Parser.AST as AST
      ')'	{ RightParenToken {} }
      '@*/'	{ CondcommentEndToken {} }
 
+     'autosemi'   { AutoSemiToken {} }
      'break'      { BreakToken {} }
      'case'       { CaseToken {} }
      'catch'      { CatchToken {} }
@@ -123,12 +124,19 @@ import qualified Language.JavaScript.Parser.AST as AST
 %%
 
 -- ---------------------------------------------------------------------
--- Sort out automatically inserted semi-colons
+-- Sort out automatically inserted semi-colons.
+-- A MaybeSemi is an actual semi-colon or nothing.
+-- An AutoSemu is either an actual semi-colon or 'virtual' semi-colon inserted
+-- by the Alex lexer.
 
 MaybeSemi :: { AST.JSNode }
 MaybeSemi : ';' { AST.NT (AST.JSLiteral ";") (ss $1) (gc $1)}
           |     { AST.NT (AST.JSLiteral "") tokenPosnEmpty []}
 
+AutoSemi :: { AST.JSNode }
+AutoSemi : ';'         { AST.NT (AST.JSLiteral ";") (ss $1) (gc $1)}
+         | 'autosemi'  { AST.NT (AST.JSLiteral "") (ss $1) (gc $1)}
+
 -- ---------------------------------------------------------------------
 
 -- Helpers
@@ -962,21 +970,21 @@ IterationStatement : Do Statement While LParen Expression RParen MaybeSemi
 --         continue [no LineTerminator here] Identifieropt ;
 -- TODO: deal with [no LineTerminator here]
 ContinueStatement :: { AST.JSNode }
-ContinueStatement : Continue MaybeSemi             { fp (AST.NN (AST.JSContinue $1 []   $2)) }
+ContinueStatement : Continue AutoSemi              { fp (AST.NN (AST.JSContinue $1 []   $2)) }
                   | Continue Identifier MaybeSemi  { fp (AST.NN (AST.JSContinue $1 [$2] $3)) }
 
 -- BreakStatement :                                                                         See 12.8
 --         break [no LineTerminator here] Identifieropt ;
 -- TODO: deal with [no LineTerminator here]
 BreakStatement :: { AST.JSNode }
-BreakStatement : Break MaybeSemi             { fp (AST.NN (AST.JSBreak $1 []   $2)) }
+BreakStatement : Break AutoSemi              { fp (AST.NN (AST.JSBreak $1 []   $2)) }
                | Break Identifier MaybeSemi  { fp (AST.NN (AST.JSBreak $1 [$2] $3)) }
 
 -- ReturnStatement :                                                                        See 12.9
 --         return [no LineTerminator here] Expressionopt ;
 -- TODO: deal with [no LineTerminator here]
 ReturnStatement :: { AST.JSNode }
-ReturnStatement : Return MaybeSemi             { fp (AST.NN (AST.JSReturn $1 []   $2)) }
+ReturnStatement : Return AutoSemi              { fp (AST.NN (AST.JSReturn $1 []   $2)) }
                 | Return Expression MaybeSemi  { fp (AST.NN (AST.JSReturn $1 [$2] $3)) }
 
 -- WithStatement :                                                                          See 12.10

src/Language/JavaScript/Parser/Lexer.x

@@ -385,13 +385,27 @@ lexCont cont = do
                 lexLoop
             WsToken {} -> do
                 addComment tok
-                lexLoop
+                ltok <- getLastToken
+                case ltok of
+                    BreakToken {} -> maybeAutoSemi tok
+                    ContinueToken {} -> maybeAutoSemi tok
+                    ReturnToken {} -> maybeAutoSemi tok
+                    _otherwise -> lexLoop
             _other -> do
                 cs <- getComment
                 let tok' = tok{ tokenComment=(toCommentAnnotation cs) }
                 setComment []
                 cont tok'
 
+    -- If the token is a WsToken and it contains a newline, convert it to an
+    -- AutoSemiToken and call the continuation, otherwise, just lexLoop.
+    maybeAutoSemi ws@(WsToken sp tl cmt) =
+        if any (== '\n') tl
+            then cont $ AutoSemiToken sp tl cmt
+            else lexLoop
+    maybeAutoSemi _ = lexLoop
+
+
 toCommentAnnotation :: [Token] -> [CommentAnnotation]
 toCommentAnnotation [] = [NoComment]
 

src/Language/JavaScript/Parser/Token.hs

@@ -94,6 +94,7 @@ data Token
 
     -- Delimiters
     -- Operators
+    | AutoSemiToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
     | SemiColonToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation]  }
     | CommaToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation]  }
     | HookToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation]  }