Fix nested indentation problem

parser/src/lexer/lexer.rs

@@ -26,6 +26,9 @@ pub struct Lexer {
     // Each time we see a right bracket we decrement this
     // This is used to match brackets in fstrings
     inside_fstring_bracket: i8,
+
+    // TODO: Hacky way to handle emitting multiple de indents
+    next_token_is_dedent: u8,
 }
 
 impl Lexer {
@@ -38,10 +41,20 @@ impl Lexer {
             nesting: 0,
             fstring_stack: vec![],
             inside_fstring_bracket: 0,
+            next_token_is_dedent: 0,
         }
     }
 
     pub fn next_token(&mut self) -> Result<Token> {
+        while self.next_token_is_dedent > 0 {
+            self.next_token_is_dedent -= 1;
+            return Ok(Token {
+                kind: Kind::Dedent,
+                value: TokenValue::None,
+                start: self.current,
+                end: self.current,
+            });
+        } 
         let start = self.current;
         let kind = self.next_kind()?;
 
@@ -68,12 +81,14 @@ impl Lexer {
         let fstring_stack = self.fstring_stack.clone();
         let start_of_line = self.start_of_line;
         let inside_fstring_bracket = self.inside_fstring_bracket;
+        let next_token_is_dedent = self.next_token_is_dedent;
         let token = self.next_token();
         self.current = current;
         self.nesting = nesting;
         self.fstring_stack = fstring_stack;
         self.start_of_line = start_of_line;
         self.inside_fstring_bracket = inside_fstring_bracket;
+        self.next_token_is_dedent = next_token_is_dedent;
         token
     }
 
@@ -843,6 +858,9 @@ impl Lexer {
                         Ordering::Less => panic!("Invalid indentation, current indentation is {} which is less than previous {}", spaces_count, top),
                     }
                 }
+                if de_indents != 1 {
+                    self.next_token_is_dedent += 1;
+                } 
                 TokenValue::Indent(de_indents)
             }
             Kind::Indent => TokenValue::Indent(1),

parser/src/lexer/snapshots/enderpy_python_parser__lexer__lexer__tests__indentation-2.snap

@@ -87,6 +87,12 @@ description: "if True:\n    if True:\n        pass\ndef"
         start: 35,
         end: 35,
     },
+    Token {
+        kind: Dedent,
+        value: None,
+        start: 35,
+        end: 35,
+    },
     Token {
         kind: Def,
         value: None,

parser/src/parser/parser.rs

@@ -167,7 +167,6 @@ impl Parser {
         }
         match token {
             Err(err) => {
-                // println!("Error: {:#?}", err);
                 self.bump_any();
             }
             Ok(token) => {
@@ -204,6 +203,31 @@ impl Parser {
         Ok(())
     }
 
+    /// Expect any of `Kinds` or return error
+    pub fn expect_any(&mut self, kind: Vec<Kind>) -> Result<(), ParsingError> {
+        if !kind.contains(&self.cur_token.kind) {
+            let found = self.cur_token.kind;
+            let node = self.start_node();
+            let range = self.finish_node(node);
+            let mut expected = String::new();
+            for kind in kind {
+                expected.push_str(&format!("{:?}, ", kind));
+            }
+            let err = ParsingError::InvalidSyntax {
+                path: Box::from(self.path.as_str()),
+                msg: Box::from(format!("Expected one of {:?} but found {:?}", expected, found)),
+                line: self.curr_line_number,
+                input: self.curr_line_string.clone(),
+                advice: "maybe you forgot to put this character".to_string(),
+                span: (range.start, range.end),
+            };
+            self.advance_to_next_line_or_semicolon();
+            return Err(err);
+        }
+        self.bump_any();
+        Ok(())
+    }
+
     fn unepxted_token(&mut self, node: Node, kind: Kind) -> Result<(), ParsingError> {
         self.bump_any();
         let range = self.finish_node(node);
@@ -400,6 +424,7 @@ impl Parser {
                 orelse = Some(if_value);
             }
         }
+
         let mut single_else_body: Option<Vec<Statement>> = None;
          if self.eat(Kind::Else) {
             self.expect(Kind::Colon)?;
@@ -422,10 +447,6 @@ impl Parser {
             }
         };
 
-        // There can be a dedent after the if block
-        // The only other token here can be a eof
-        self.bump(Kind::Dedent);
-
         Ok(Statement::IfStatement(If {
             node: self.finish_node(node),
             test,
@@ -753,6 +774,7 @@ impl Parser {
         self.expect(Kind::NewLine)?;
         self.expect(Kind::Indent)?;
         let cases = self.parse_cases()?;
+        self.expect_any(vec![Kind::Dedent, Kind::Eof])?;
 
         Ok(Statement::Match(Match {
             node: self.finish_node(node),
@@ -2248,6 +2270,7 @@ impl Parser {
         } else if is_atom(&self.cur_kind()) {
             self.parse_atom()?
         } else {
+            panic!("invalid primary expression {:?}", self.cur_kind());
             return Err(self.unepxted_token(node, self.cur_kind()).err().unwrap());
         };
 
@@ -3656,68 +3679,6 @@ class a: pass",
         }
     }
 
-    #[test]
-    fn test_match_statement() {
-        for test_case in &[
-            "match a:
-    case 1:
-        pass",
-            "match a:
-    case 1 | 2:
-        pass",
-            "match a.b:
-    case 1:
-        pass",
-            "match a:
-    case None:
-        pass
-    case True:
-        pass
-    case False:
-        pass
-    case -1:
-        pass
-    case 1.0:
-        pass
-    case _:
-        pass
-",
-            "match a:
-    case a.b:
-        pass
-    case a:
-        pass
-",
-            "match a:
-    case (a, b):
-        pass
-    case {1: _, 2: _}:
-        pass
-    case {**rest}:
-        pass
-",
-            "match x:
-    case Point2D(0, 0):
-        pass
-    case Point3D(x=0, y=0, z=0):
-        pass
-",
-            "match x:
-    case [a, b, c]:
-        pass",
-        ] {
-            let mut parser = Parser::new(test_case.to_string(), String::from(""));
-            let program = parser.parse();
-
-            insta::with_settings!({
-                    description => test_case.to_string(), // the template source code
-                    omit_expression => true // do not include the default expression
-                }, {
-                    assert_debug_snapshot!(program);
-            });
-        }
-    }
-
     #[test]
     fn test_complete() {
         glob!("../../test_data", "inputs/*.py", |path| {