fix: handle new lines following each other

Author: Glyphack

parser/src/lexer/mod.rs

@@ -63,7 +63,7 @@ pub struct Lexer<'a> {
     peak_mode: bool,
 
     /// Previous token was a Newline token
-    prev_token_newline: bool,
+    non_logical_line_state: bool,
     /// Cursor at position after the indentation in line
     indented: bool,
 }
@@ -81,7 +81,7 @@ impl<'a> Lexer<'a> {
             next_token_is_dedent: 0,
             line_starts: vec![0],
             peak_mode: false,
-            prev_token_newline: true,
+            non_logical_line_state: true,
             indented: false,
         }
     }
@@ -132,7 +132,9 @@ impl<'a> Lexer<'a> {
             return self.next_token();
         }
 
-        self.prev_token_newline = kind == Kind::NewLine;
+        if kind != Kind::Comment && kind != Kind::NL {
+            self.non_logical_line_state = kind == Kind::NewLine;
+        }
         let value = self.parse_token_value(kind, start);
         let end = self.current;
 
@@ -155,12 +157,12 @@ impl<'a> Lexer<'a> {
         let nesting = self.nesting;
         let start_of_line = self.start_of_line;
         let next_token_is_dedent = self.next_token_is_dedent;
-        let prev_token_newline = self.prev_token_newline;
+        let prev_token_newline = self.non_logical_line_state;
         let indented = self.indented;
         self.peak_mode = true;
         let token = self.next_token();
         self.indented = indented;
-        self.prev_token_newline = prev_token_newline;
+        self.non_logical_line_state = prev_token_newline;
         self.peak_mode = false;
         self.current = current;
         self.current_line = current_line;
@@ -527,7 +529,7 @@ impl<'a> Lexer<'a> {
                 '\n' | '\r' => {
                     self.current_line += 1;
                     self.start_of_line = true;
-                    if self.nesting == 0 && (!self.prev_token_newline) {
+                    if self.nesting == 0 && (!self.non_logical_line_state) {
                         return Ok(Kind::NewLine);
                     } else {
                         return Ok(Kind::NL);
@@ -1069,19 +1071,6 @@ impl<'a> Lexer<'a> {
         }
         count
     }
-
-    // This function is used in test. Not sure to keep it here or not
-    #[allow(dead_code)]
-    fn to_row_col(&self, source_offset: u32) -> (u32, u32) {
-        println!("source_offset: {}", source_offset);
-        println!("line_starts: {:?}", self.line_starts);
-        let (line_row, line_offset) = match self.line_starts.binary_search(&source_offset) {
-            Ok(idx) => (idx, self.line_starts[idx]),
-            Err(idx) => (idx - 1, self.line_starts[idx - 1]),
-        };
-        let line_column = source_offset - line_offset;
-        (u32::try_from(line_row).unwrap(), line_column)
-    }
 }
 
 fn match_whitespace(c: char) -> bool {

parser/src/parser/parser.rs

@@ -168,7 +168,7 @@ impl<'a> Parser<'a> {
 
         self.prev_token_end = self.cur_token.end;
         self.cur_token = token;
-        if self.cur_kind() == Kind::Comment {
+        if matches!(self.cur_kind(), Kind::Comment | Kind::NL) {
             self.advance();
         }
         if self.nested_expression_list > 0 {
@@ -4047,7 +4047,7 @@ except *Exception as e:
                 let snapshot = format!("{program:#?}");
 
                 insta::with_settings!({
-                        description => test_case.clone(),
+                        description => format!("test file: {}\n{}", $test_file, test_case.clone()),
                         omit_expression => true,
                         snapshot_path => "../../test_data/output/"
                     }, {

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap

@@ -14,5 +14,5 @@ description: "if a:\n\n    f = c\n\n    # Path: test_local.py\n"
 16,17: NewLine   (None)
 17,18: NL   (None)
 22,43: Comment   (Str("# Path: test_local.py"))
-43,44: NewLine   (None)
+43,44: NL   (None)
 44,44: Dedent   (Indent(1))

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap

@@ -4,14 +4,14 @@ description: "# a\n# a\n\n# a\n# b\ndef a(): ... # a\n\ndef b():\n    # a\n    .
 input_file: parser/test_data/inputs/comments.py
 ---
 0,3: Comment   (Str("# a"))
-3,4: NewLine   (None)
+3,4: NL   (None)
 4,7: Comment   (Str("# a"))
-7,8: NewLine   (None)
+7,8: NL   (None)
 8,9: NL   (None)
 9,12: Comment   (Str("# a"))
-12,13: NewLine   (None)
+12,13: NL   (None)
 13,16: Comment   (Str("# b"))
-16,17: NewLine   (None)
+16,17: NL   (None)
 17,20: Def   (None)
 21,22: Identifier   (Str("a"))
 22,23: (   (None)

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@from_import.py.snap

@@ -95,6 +95,6 @@ input_file: parser/test_data/inputs/from_import.py
 244,245: Identifier   (Str("b"))
 245,246: NewLine   (None)
 246,289: Comment   (Str("# TODO(parser): enable after error handling"))
-289,290: NewLine   (None)
+289,290: NL   (None)
 290,300: Comment   (Str("# from ..."))
-300,301: NewLine   (None)
+300,301: NL   (None)

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap

@@ -62,7 +62,7 @@ input_file: parser/test_data/inputs/functions.py
 327,339: Identifier   (Str("ticker_index"))
 339,340: NewLine   (None)
 340,341: NL   (None)
-341,342: NewLine   (None)
+341,342: NL   (None)
 342,342: Dedent   (Indent(1))
 342,345: Def   (None)
 346,379: Identifier   (Str("_extract_ticker_client_types_data"))

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap

@@ -98,7 +98,7 @@ input_file: parser/test_data/inputs/if.py
 250,254: Pass   (None)
 254,255: NewLine   (None)
 255,256: NL   (None)
-256,257: NewLine   (None)
+256,257: NL   (None)
 257,257: Dedent   (Indent(1))
 257,259: If   (None)
 260,261: Identifier   (Str("a"))

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@indentation.py.snap

@@ -4,10 +4,10 @@ description: "# Unindent does not match any outer indentation level\n# if True:\
 input_file: parser/test_data/inputs/indentation.py
 ---
 0,53: Comment   (Str("# Unindent does not match any outer indentation level"))
-53,54: NewLine   (None)
+53,54: NL   (None)
 54,64: Comment   (Str("# if True:"))
-64,65: NewLine   (None)
+64,65: NL   (None)
 65,79: Comment   (Str("#         pass"))
-79,80: NewLine   (None)
+79,80: NL   (None)
 80,90: Comment   (Str("#     pass"))
-90,91: NewLine   (None)
+90,91: NL   (None)

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap

@@ -40,7 +40,7 @@ input_file: parser/test_data/inputs/newlines.py
 64,65: ]   (None)
 65,66: NewLine   (None)
 66,67: NL   (None)
-67,68: NewLine   (None)
+67,68: NL   (None)
 68,71: Def   (None)
 72,77: Identifier   (Str("hello"))
 77,78: (   (None)
@@ -56,7 +56,7 @@ input_file: parser/test_data/inputs/newlines.py
 96,99: Ellipsis   (None)
 99,100: NewLine   (None)
 100,101: NL   (None)
-101,102: NewLine   (None)
+101,102: NL   (None)
 102,107: Class   (None)
 108,109: Identifier   (Str("A"))
 109,110: (   (None)

parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap

@@ -4,7 +4,7 @@ description: "# Test case to check that we return correct error when two python
 input_file: parser/test_data/inputs/separate_statements.py
 ---
 0,97: Comment   (Str("# Test case to check that we return correct error when two python statements are on the same line"))
-97,98: NewLine   (None)
+97,98: NL   (None)
 98,99: NL   (None)
 99,102: Def   (None)
 103,106: Identifier   (Str("foo"))
@@ -38,7 +38,7 @@ input_file: parser/test_data/inputs/separate_statements.py
 160,203: Comment   (Str("# TODO(parser): enable after error handling"))
 203,204: NewLine   (None)
 208,221: Comment   (Str("# x = 1 y = 2"))
-221,222: NewLine   (None)
+221,222: NL   (None)
 226,232: Return   (None)
 233,234: Identifier   (Str("x"))
 235,236: +   (None)
@@ -49,4 +49,4 @@ input_file: parser/test_data/inputs/separate_statements.py
 240,283: Comment   (Str("# TODO(parser): enable after error handling"))
 283,284: NewLine   (None)
 284,298: Comment   (Str("# a = 1  b = 2"))
-298,299: NewLine   (None)
+298,299: NL   (None)