Skip to content

Commit

Permalink
(parser): refactor comments and indentation
Browse files Browse the repository at this point in the history
  • Loading branch information
@Glyphack
Glyphack committed Sep 24, 2023
1 parent 7d3afb1 commit af7db4a
Show file tree
Hide file tree
Showing 8 changed files with 335 additions and 28 deletions.
47 changes: 43 additions & 4 deletions parser/src/lexer/lexer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,16 @@ impl Lexer {
}
_ => return Ok(Kind::Assign),
},
'#' => return Ok(Kind::Sharp),
'#' => {
// consume until new line
while let Some(c) = self.peek() {
if c == '\n' || c == '\r' {
break;
}
self.next();
}
return Ok(Kind::Comment);
},
'\\' => return Ok(Kind::BackSlash),
'$' => return Ok(Kind::Dollar),
'?' => return Ok(Kind::QuestionMark),
Expand Down Expand Up @@ -754,10 +763,28 @@ impl Lexer {
}
}
}
if spaces_count == 0 {
// When there are no spaces and only a new line
// like the following
// if True:
//
// print("Hello")
//
// We should not consider that new line as dedent
//
// But also if this part is the last part of the file
// we should not consider it as dedent
// Thanks python
if self.peek() == Some('\n') && self.double_peek().is_some() {
return None
}
}
println!("spaces_count: {}", spaces_count);
if let Some(top) = self.indent_stack.last() {
match spaces_count.cmp(top) {
Ordering::Less => Some(Kind::Dedent),
Ordering::Equal => None,
// Returning whitespace to ignore these spaces
Ordering::Equal => Some(Kind::WhiteSpace),
Ordering::Greater => {
self.indent_stack.push(spaces_count);
Some(Kind::Indent)
Expand Down Expand Up @@ -789,7 +816,8 @@ impl Lexer {
| Kind::RawString
| Kind::RawFStringStart
| Kind::Bytes
| Kind::Unicode => TokenValue::Str(kind_value),
| Kind::Unicode
| Kind::Comment => TokenValue::Str(kind_value),
Kind::Dedent => {
let mut spaces_count = 0;
for c in kind_value.chars() {
Expand Down Expand Up @@ -1088,7 +1116,7 @@ mod tests {
"indentation",
&[
"if True:
pass",
pass\n",
"if True:
pass
else:
Expand All @@ -1097,6 +1125,17 @@ else:
if True:
pass
def",
"def f(x):
y = z
print(y)
",
"if a:
f = c
# Path: test_local.py
"
],
)
.unwrap();
Expand Down
16 changes: 15 additions & 1 deletion parser/src/lexer/snapshots/enderpy_python_parser__lexer__lexer__tests__indentation-0.snap
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
source: parser/src/lexer/lexer.rs
description: "if True:\n pass"
description: "if True:\n pass\n"
---
[
Token {
Expand Down Expand Up @@ -41,4 +41,18 @@ description: "if True:\n pass"
start: 21,
end: 25,
},
Token {
kind: NewLine,
value: None,
start: 25,
end: 26,
},
Token {
kind: Dedent,
value: Indent(
1,
),
start: 26,
end: 26,
},
]
136 changes: 136 additions & 0 deletions parser/src/lexer/snapshots/enderpy_python_parser__lexer__lexer__tests__indentation-3.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
---
source: parser/src/lexer/lexer.rs
description: "def f(x):\n y = z\n\n print(y)\n"
---
[
Token {
kind: Def,
value: None,
start: 0,
end: 3,
},
Token {
kind: Identifier,
value: Str(
"f",
),
start: 4,
end: 5,
},
Token {
kind: LeftParen,
value: None,
start: 5,
end: 6,
},
Token {
kind: Identifier,
value: Str(
"x",
),
start: 6,
end: 7,
},
Token {
kind: RightParen,
value: None,
start: 7,
end: 8,
},
Token {
kind: Colon,
value: None,
start: 8,
end: 9,
},
Token {
kind: NewLine,
value: None,
start: 9,
end: 10,
},
Token {
kind: Indent,
value: Indent(
1,
),
start: 10,
end: 14,
},
Token {
kind: Identifier,
value: Str(
"y",
),
start: 14,
end: 15,
},
Token {
kind: Assign,
value: None,
start: 16,
end: 17,
},
Token {
kind: Identifier,
value: Str(
"z",
),
start: 18,
end: 19,
},
Token {
kind: NewLine,
value: None,
start: 19,
end: 20,
},
Token {
kind: NewLine,
value: None,
start: 20,
end: 21,
},
Token {
kind: Identifier,
value: Str(
"print",
),
start: 25,
end: 30,
},
Token {
kind: LeftParen,
value: None,
start: 30,
end: 31,
},
Token {
kind: Identifier,
value: Str(
"y",
),
start: 31,
end: 32,
},
Token {
kind: RightParen,
value: None,
start: 32,
end: 33,
},
Token {
kind: NewLine,
value: None,
start: 33,
end: 34,
},
Token {
kind: Dedent,
value: Indent(
1,
),
start: 34,
end: 34,
},
]
102 changes: 102 additions & 0 deletions parser/src/lexer/snapshots/enderpy_python_parser__lexer__lexer__tests__indentation-4.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
---
source: parser/src/lexer/lexer.rs
description: "if a:\n\n f = c\n\n # Path: test_local.py\t\n"
---
[
Token {
kind: If,
value: None,
start: 0,
end: 2,
},
Token {
kind: Identifier,
value: Str(
"a",
),
start: 3,
end: 4,
},
Token {
kind: Colon,
value: None,
start: 4,
end: 5,
},
Token {
kind: NewLine,
value: None,
start: 5,
end: 6,
},
Token {
kind: NewLine,
value: None,
start: 6,
end: 7,
},
Token {
kind: Indent,
value: Indent(
1,
),
start: 7,
end: 11,
},
Token {
kind: Identifier,
value: Str(
"f",
),
start: 11,
end: 12,
},
Token {
kind: Assign,
value: None,
start: 13,
end: 14,
},
Token {
kind: Identifier,
value: Str(
"c",
),
start: 15,
end: 16,
},
Token {
kind: NewLine,
value: None,
start: 16,
end: 17,
},
Token {
kind: NewLine,
value: None,
start: 17,
end: 18,
},
Token {
kind: Comment,
value: Str(
"# Path: test_local.py\t",
),
start: 22,
end: 44,
},
Token {
kind: NewLine,
value: None,
start: 44,
end: 45,
},
Token {
kind: Dedent,
value: Indent(
1,
),
start: 45,
end: 45,
},
]
6 changes: 4 additions & 2 deletions parser/src/lexer/snapshots/enderpy_python_parser__lexer__lexer__tests__test-lexer-15.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ description: "#"
---
[
Token {
kind: Sharp,
value: None,
kind: Comment,
value: Str(
"#",
),
start: 0,
end: 1,
},
Expand Down
3 changes: 2 additions & 1 deletion parser/src/parser/diagnostics.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ pub struct ExpectToken(
#[error("Unexpected token")]
#[diagnostic()]
pub struct UnexpectedToken(
pub u32,
pub &'static str,
#[label("Unexpected token `{0}`")] pub Node,
#[label("line: {0} Unexpected token `{1}`")] pub Node,
);

#[derive(Debug, Error, Diagnostic)]
Expand Down
Loading

0 comments on commit af7db4a

Please sign in to comment.