From 3a82579062d3fa7506b7625101d33eb106a893e8 Mon Sep 17 00:00:00 2001 From: Shaygan Hooshyari Date: Wed, 9 Oct 2024 21:31:05 +0200 Subject: [PATCH] Remove token value (#267) * Remove token value * Update parser tests * Remove extra brackets in fstring middle * Fix compatibility tests * Add test for fstring middle with brackets --- benchmark/benches/parser_benchmark.rs | 2 +- compat/src/lexer_compat.rs | 10 +- compat/src/parser_compat.rs | 2 +- enderpy/src/main.rs | 5 +- parser/src/error.rs | 24 - parser/src/lexer/mod.rs | 66 +- parser/src/parser/mod.rs | 15 +- parser/src/parser/parser.rs | 361 +++---- parser/src/token.rs | 110 +-- ..._snapshot_test_lexer@bytes-literals-0.snap | 2 +- ..._snapshot_test_lexer@bytes-literals-1.snap | 2 +- ..._snapshot_test_lexer@bytes-literals-2.snap | 2 +- ..._snapshot_test_lexer@bytes-literals-3.snap | 6 +- ..._snapshot_test_lexer@bytes-literals-4.snap | 2 +- ..._snapshot_test_lexer@bytes-literals-5.snap | 2 +- ..._snapshot_test_lexer@bytes-literals-6.snap | 2 +- ...tests__snapshot_test_lexer@ellipsis-0.snap | 2 +- ...tests__snapshot_test_lexer@ellipsis-1.snap | 16 +- ...apshot_test_lexer@f-string-literals-0.snap | 6 +- ...apshot_test_lexer@f-string-literals-1.snap | 12 +- ...pshot_test_lexer@f-string-literals-10.snap | 20 +- ...apshot_test_lexer@f-string-literals-2.snap | 6 +- ...apshot_test_lexer@f-string-literals-3.snap | 4 +- ...apshot_test_lexer@f-string-literals-4.snap | 10 +- ...apshot_test_lexer@f-string-literals-5.snap | 6 +- ...apshot_test_lexer@f-string-literals-6.snap | 6 +- ...apshot_test_lexer@f-string-literals-7.snap | 6 +- ...apshot_test_lexer@f-string-literals-8.snap | 6 +- ...apshot_test_lexer@f-string-literals-9.snap | 8 +- ..._snapshot_test_lexer@fstring_escape-0.snap | 6 +- ..._snapshot_test_lexer@fstring_escape-1.snap | 24 +- ...ts__snapshot_test_lexer@identifiers-0.snap | 2 +- ...ts__snapshot_test_lexer@identifiers-1.snap | 2 +- ...ts__snapshot_test_lexer@identifiers-2.snap | 2 +- ...ts__snapshot_test_lexer@identifiers-3.snap | 2 +- ...ts__snapshot_test_lexer@identifiers-4.snap | 2 +- ...ts__snapshot_test_lexer@identifiers-5.snap | 2 +- ...ts__snapshot_test_lexer@identifiers-6.snap | 2 +- ...__tests__snapshot_test_lexer@import-0.snap | 4 +- ...__tests__snapshot_test_lexer@import-1.snap | 8 +- ...__tests__snapshot_test_lexer@import-2.snap | 12 +- ...__tests__snapshot_test_lexer@import-3.snap | 8 +- ...ts__snapshot_test_lexer@indentation-0.snap | 16 +- ...ts__snapshot_test_lexer@indentation-1.snap | 26 +- ...ts__snapshot_test_lexer@indentation-2.snap | 30 +- ...ts__snapshot_test_lexer@indentation-3.snap | 38 +- ...ts__snapshot_test_lexer@indentation-4.snap | 28 +- ...tests__snapshot_test_lexer@keywords-0.snap | 16 +- ...tests__snapshot_test_lexer@keywords-1.snap | 16 +- ...tests__snapshot_test_lexer@keywords-2.snap | 18 +- ...tests__snapshot_test_lexer@keywords-3.snap | 20 +- ...t_lexer@newline-in-nested-structure-0.snap | 12 +- ..._snapshot_test_lexer@numeric-binary-0.snap | 2 +- ..._snapshot_test_lexer@numeric-binary-1.snap | 2 +- ..._snapshot_test_lexer@numeric-binary-2.snap | 2 +- ..._snapshot_test_lexer@numeric-binary-3.snap | 2 +- ..._snapshot_test_lexer@numeric-binary-4.snap | 2 +- ..._snapshot_test_lexer@numeric-binary-5.snap | 2 +- ..._snapshot_test_lexer@numeric-binary-6.snap | 2 +- ..._snapshot_test_lexer@numeric-binary-7.snap | 2 +- ...est_lexer@numeric-floating-exponent-0.snap | 10 +- ...t_test_lexer@numeric-floating-point-0.snap | 12 +- ...shot_test_lexer@numeric-hexadecimal-0.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-1.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-10.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-11.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-12.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-13.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-14.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-15.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-16.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-17.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-18.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-19.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-2.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-20.snap | 2 +- ...hot_test_lexer@numeric-hexadecimal-21.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-3.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-4.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-5.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-6.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-7.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-8.snap | 2 +- ...shot_test_lexer@numeric-hexadecimal-9.snap | 2 +- ...snapshot_test_lexer@numeric-integer-0.snap | 8 +- ...__snapshot_test_lexer@numeric-octal-0.snap | 2 +- ...__snapshot_test_lexer@numeric-octal-1.snap | 2 +- ...__snapshot_test_lexer@numeric-octal-2.snap | 2 +- ...__snapshot_test_lexer@numeric-octal-3.snap | 2 +- ...__snapshot_test_lexer@numeric-octal-4.snap | 2 +- ...__snapshot_test_lexer@numeric-octal-5.snap | 2 +- ...__snapshot_test_lexer@numeric-octal-6.snap | 2 +- ...__snapshot_test_lexer@numeric-octal-7.snap | 2 +- ...pshot_test_lexer@raw-bytes-literals-0.snap | 2 +- ...pshot_test_lexer@raw-bytes-literals-1.snap | 2 +- ...pshot_test_lexer@raw-bytes-literals-2.snap | 2 +- ...pshot_test_lexer@raw-bytes-literals-3.snap | 6 +- ...pshot_test_lexer@raw-bytes-literals-4.snap | 2 +- ...pshot_test_lexer@raw-bytes-literals-5.snap | 2 +- ...pshot_test_lexer@raw-bytes-literals-6.snap | 2 +- ...ot_test_lexer@raw-f-string-literals-0.snap | 6 +- ...ot_test_lexer@raw-f-string-literals-1.snap | 6 +- ...ot_test_lexer@raw-f-string-literals-2.snap | 4 +- ...ot_test_lexer@raw-f-string-literals-3.snap | 10 +- ...ot_test_lexer@raw-f-string-literals-4.snap | 12 +- ...ot_test_lexer@raw-f-string-literals-5.snap | 6 +- ...ot_test_lexer@raw-f-string-literals-6.snap | 6 +- ...shot_test_lexer@raw-string-literals-0.snap | 2 +- ...shot_test_lexer@raw-string-literals-1.snap | 2 +- ...shot_test_lexer@raw-string-literals-2.snap | 2 +- ...shot_test_lexer@raw-string-literals-3.snap | 6 +- ...shot_test_lexer@raw-string-literals-4.snap | 2 +- ...shot_test_lexer@raw-string-literals-5.snap | 2 +- ...shot_test_lexer@raw-string-literals-6.snap | 2 +- ...snapshot_test_lexer@string-literals-0.snap | 2 +- ...snapshot_test_lexer@string-literals-1.snap | 2 +- ...snapshot_test_lexer@string-literals-2.snap | 2 +- ...snapshot_test_lexer@string-literals-3.snap | 6 +- ...snapshot_test_lexer@string-literals-4.snap | 2 +- ...snapshot_test_lexer@string-literals-5.snap | 2 +- ...snapshot_test_lexer@string-literals-6.snap | 2 +- ...snapshot_test_lexer@string-literals-7.snap | 2 +- ...sts__snapshot_test_lexer@test-lexer-0.snap | 6 +- ...sts__snapshot_test_lexer@test-lexer-1.snap | 6 +- ...ts__snapshot_test_lexer@test-lexer-10.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-11.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-12.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-13.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-14.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-15.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-16.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-17.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-18.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-19.snap | 2 +- ...sts__snapshot_test_lexer@test-lexer-2.snap | 6 +- ...ts__snapshot_test_lexer@test-lexer-20.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-21.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-22.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-23.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-24.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-25.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-26.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-27.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-28.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-29.snap | 2 +- ...sts__snapshot_test_lexer@test-lexer-3.snap | 4 +- ...ts__snapshot_test_lexer@test-lexer-30.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-31.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-32.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-33.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-34.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-35.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-36.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-37.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-38.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-39.snap | 2 +- ...sts__snapshot_test_lexer@test-lexer-4.snap | 6 +- ...ts__snapshot_test_lexer@test-lexer-40.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-41.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-42.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-43.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-44.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-45.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-46.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-47.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-48.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-49.snap | 2 +- ...sts__snapshot_test_lexer@test-lexer-5.snap | 6 +- ...ts__snapshot_test_lexer@test-lexer-50.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-51.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-52.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-53.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-54.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-55.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-56.snap | 2 +- ...ts__snapshot_test_lexer@test-lexer-57.snap | 2 +- ...sts__snapshot_test_lexer@test-lexer-6.snap | 4 +- ...sts__snapshot_test_lexer@test-lexer-7.snap | 4 +- ...sts__snapshot_test_lexer@test-lexer-8.snap | 6 +- ...sts__snapshot_test_lexer@test-lexer-9.snap | 2 +- ..._test_lexer@unicode-string-literals-0.snap | 2 +- ..._test_lexer@unicode-string-literals-1.snap | 2 +- ..._test_lexer@unicode-string-literals-2.snap | 2 +- ..._test_lexer@unicode-string-literals-3.snap | 6 +- ..._test_lexer@unicode-string-literals-4.snap | 2 +- ..._test_lexer@unicode-string-literals-5.snap | 2 +- ..._test_lexer@unicode-string-literals-6.snap | 2 +- ...er_and_errors@annotated_assignment.py.snap | 208 ++--- ...ot_test_lexer_and_errors@binary_op.py.snap | 142 +-- ...apshot_test_lexer_and_errors@class.py.snap | 504 +++++----- ...hot_test_lexer_and_errors@comments.py.snap | 68 +- ...napshot_test_lexer_and_errors@dict.py.snap | 318 +++---- ...snapshot_test_lexer_and_errors@for.py.snap | 216 ++--- ..._test_lexer_and_errors@from_import.py.snap | 190 ++-- ...test_lexer_and_errors@function_def.py.snap | 878 +++++++++--------- ...ot_test_lexer_and_errors@functions.py.snap | 556 +++++------ ...r_and_errors@generator_expressions.py.snap | 254 ++--- ..._snapshot_test_lexer_and_errors@if.py.snap | 314 +++---- ..._test_lexer_and_errors@indentation.py.snap | 16 +- ...apshot_test_lexer_and_errors@lists.py.snap | 316 +++---- ...apshot_test_lexer_and_errors@match.py.snap | 474 +++++----- ...hot_test_lexer_and_errors@newlines.py.snap | 136 +-- ...xer_and_errors@separate_statements.py.snap | 94 +- ...napshot_test_lexer_and_errors@sets.py.snap | 378 ++++---- ...pshot_test_lexer_and_errors@string.py.snap | 260 +++--- ...ot_test_lexer_and_errors@subscript.py.snap | 306 +++--- ...snapshot_test_lexer_and_errors@try.py.snap | 472 +++++----- ...t_test_lexer_and_errors@type_alias.py.snap | 42 +- ...napshot_test_lexer_and_errors@with.py.snap | 172 ++-- typechecker/src/file.rs | 15 +- typechecker/src/type_evaluator.rs | 2 +- 211 files changed, 3788 insertions(+), 3936 deletions(-) diff --git a/benchmark/benches/parser_benchmark.rs b/benchmark/benches/parser_benchmark.rs index 66769e9c..8cb866a1 100644 --- a/benchmark/benches/parser_benchmark.rs +++ b/benchmark/benches/parser_benchmark.rs @@ -56,7 +56,7 @@ pub fn benchmark_parser(c: &mut Criterion) { &source, |b, source| { b.iter(|| { - let mut parser = Parser::new(source, path); + let mut parser = Parser::new(source); parser.parse().unwrap(); 0 diff --git a/compat/src/lexer_compat.rs b/compat/src/lexer_compat.rs index 53942aae..14778e7b 100644 --- a/compat/src/lexer_compat.rs +++ b/compat/src/lexer_compat.rs @@ -430,7 +430,7 @@ fn check_tokens_match( } let python_token_value = python_token.value.clone(); - let enderpy_token_value = enderpy_token.value.to_string(); + let enderpy_token_value = enderpy_token.to_string(lexer.source); // The Python tokenizer sets values in a number of places where Enderpy simply relies // on kind to assume value. Handle those cases here. let value_matches = matches_python_name_token(python_token.value.as_str(), &enderpy_token.kind) @@ -507,6 +507,8 @@ fn matches_python_name_token(python_token_value: &str, token_kind: &Kind) -> boo "while" => token_kind == &Kind::While, "with" => token_kind == &Kind::With, "yield" => token_kind == &Kind::Yield, + "match" => token_kind == &Kind::Match, + "type" => token_kind == &Kind::Type, _ => token_kind == &Kind::Identifier, } } @@ -903,6 +905,12 @@ print(a) ]); } + // TODO: fstring middle offset is wrong in case of {{ or }} + #[test] + fn test_fstring_positions() { + python_tokenize_test_lexer(&["f\"{{{', '.join(dict_items)}}}\""]); + } + #[test] #[should_panic] fn test_lex_unterminated_string_double_quotes() { diff --git a/compat/src/parser_compat.rs b/compat/src/parser_compat.rs index 5893f8b1..3f670d55 100644 --- a/compat/src/parser_compat.rs +++ b/compat/src/parser_compat.rs @@ -110,7 +110,7 @@ fn remove_unimplemented_attributes(value: &mut Value) { } fn parse_enderpy_source(source: &str) -> Result { - let mut parser = Parser::new(source, "string"); + let mut parser = Parser::new(source); let typed_ast = parser.parse().into_diagnostic()?; let ast = typed_ast.as_python_compat(&parser); Ok(ast) diff --git a/enderpy/src/main.rs b/enderpy/src/main.rs index ed2d72f3..ab73022b 100644 --- a/enderpy/src/main.rs +++ b/enderpy/src/main.rs @@ -91,13 +91,12 @@ fn tokenize() -> Result<()> { let (start_line_num, start_line_column, end_line_num, end_line_column) = get_row_col_position(token.start, token.end, &lexer.line_starts); println!( - "{}-{}, {}-{}: {} {} {} {}", + "{}-{}, {}-{}: {} {} {}", start_line_num, start_line_column, end_line_num, end_line_column, token.kind, - token.value, token.start, token.end, ); @@ -108,7 +107,7 @@ fn tokenize() -> Result<()> { fn parse(file: &PathBuf) -> Result<()> { let source = fs::read_to_string(file).into_diagnostic()?; let file_path = file.to_str().unwrap_or(""); - let mut parser = Parser::new(&source, file_path); + let mut parser = Parser::new(&source); let ast = parser.parse(); println!("{:#?}", ast); Ok(()) diff --git a/parser/src/error.rs b/parser/src/error.rs index 32bb946b..0d2d4902 100644 --- a/parser/src/error.rs +++ b/parser/src/error.rs @@ -1,8 +1,6 @@ use miette::Diagnostic; use thiserror::Error; -use crate::parser::parser::Parser; - #[derive(Error, Diagnostic, Debug, Clone)] pub enum ParsingError { #[error("Invalid syntax")] @@ -16,28 +14,6 @@ pub enum ParsingError { }, } -impl From> for ParsingError { - fn from(err: Parser) -> Self { - let token = err.cur_token(); - ParsingError::InvalidSyntax { - msg: token.value.to_string(), - advice: String::default(), - span: err.get_span_on_line(token.start, token.end), - } - } -} - -impl From<&mut Parser<'_>> for ParsingError { - fn from(err: &mut Parser) -> Self { - let token = err.cur_token(); - ParsingError::InvalidSyntax { - msg: token.value.to_string(), - advice: String::default(), - span: err.get_span_on_line(token.start, token.end), - } - } -} - #[derive(Error, Debug)] pub enum LexError { #[error("String not terminated")] diff --git a/parser/src/lexer/mod.rs b/parser/src/lexer/mod.rs index a4f48794..a790990c 100644 --- a/parser/src/lexer/mod.rs +++ b/parser/src/lexer/mod.rs @@ -3,7 +3,7 @@ use unicode_id_start::{is_id_continue, is_id_start}; use crate::{ error::LexError, get_row_col_position, - token::{Kind, Token, TokenValue}, + token::{Kind, Token}, }; #[derive(Debug, Clone, Copy, PartialEq)] @@ -101,7 +101,6 @@ impl<'a> Lexer<'a> { self.next_token_is_dedent -= 1; return Token { kind: Kind::Dedent, - value: TokenValue::None, start: self.current, end: self.current, }; @@ -138,22 +137,12 @@ impl<'a> Lexer<'a> { if kind != Kind::Comment && kind != Kind::NL && kind != Kind::Dedent { self.non_logical_line_state = kind == Kind::NewLine; } - let value = self.parse_token_value(kind, start); - let end = match kind { - Kind::FStringMiddle => start + value.as_str().expect("").len() as u32, - _ => self.current, - }; - + let end = self.current; if kind == Kind::Dedent { start = end } - Token { - kind, - value, - start, - end, - } + Token { kind, start, end } } // peek_token is a side-effect free version of next_token @@ -750,6 +739,8 @@ impl<'a> Lexer<'a> { "while" => Kind::While, "with" => Kind::With, "yield" => Kind::Yield, + "match" => Kind::Match, + "type" => Kind::Type, _ => Kind::Identifier, } } @@ -1053,43 +1044,6 @@ impl<'a> Lexer<'a> { } } - fn parse_token_value(&mut self, kind: Kind, start: u32) -> TokenValue { - let kind_value = &self.source[start as usize..self.current as usize]; - match kind { - Kind::Integer - | Kind::Hexadecimal - | Kind::Binary - | Kind::PointFloat - | Kind::Octal - | Kind::ExponentFloat - | Kind::ImaginaryInteger - | Kind::ImaginaryExponentFloat - | Kind::ImaginaryPointFloat => TokenValue::Number(kind_value.to_string()), - Kind::Identifier => match kind_value { - "type" => TokenValue::Type, - "match" => TokenValue::Match, - _ => TokenValue::Str(kind_value.to_string()), - }, - Kind::StringLiteral - | Kind::FStringStart - | Kind::FStringEnd - | Kind::RawBytes - | Kind::RawFStringStart - | Kind::Bytes - | Kind::Unicode - | Kind::Comment => TokenValue::Str(kind_value.to_string()), - Kind::FStringMiddle => { - let value = kind_value.replace("{{", "{"); - let value = value.replace("}}", "}"); - TokenValue::Str(value) - } - Kind::Dedent => TokenValue::Indent(1), - Kind::Indent => TokenValue::Indent(1), - Kind::Error => TokenValue::Str(kind_value.to_string()), - _ => TokenValue::None, - } - } - fn f_string_quote_count(&mut self, str_start: char) -> u8 { let mut count = 1; if self.peek() == Some(str_start) && self.double_peek() == Some(str_start) { @@ -1115,15 +1069,14 @@ mod tests { fn snapshot_test_lexer_and_errors(test_case: &str) { let mut lexer = Lexer::new(test_case); - let mut tokens = vec![]; let mut snapshot = String::from(""); loop { let token = lexer.next_token(); if token.kind == Kind::Eof { break; } - snapshot += format!("{}\n", token).as_str(); - tokens.push(token); + snapshot += token.display_token(test_case).as_str(); + snapshot += "\n"; } let mut settings = insta::Settings::clone_current(); settings.set_snapshot_path("../../test_data/output/"); @@ -1136,15 +1089,14 @@ mod tests { fn snapshot_test_lexer(snap_name: &str, inputs: &[&str]) -> Result<(), LexError> { for (i, test_input) in inputs.iter().enumerate() { let mut lexer = Lexer::new(test_input); - let mut tokens = vec![]; let mut snapshot = String::from(""); loop { let token = lexer.next_token(); if token.kind == Kind::Eof { break; } - snapshot += format!("{}\n", token).as_str(); - tokens.push(token); + snapshot += token.display_token(test_input).as_str(); + snapshot += "\n"; } let mut settings = insta::Settings::clone_current(); settings.set_snapshot_suffix(format!("{snap_name}-{i}")); diff --git a/parser/src/parser/mod.rs b/parser/src/parser/mod.rs index 3c638c55..932f23f0 100644 --- a/parser/src/parser/mod.rs +++ b/parser/src/parser/mod.rs @@ -11,7 +11,7 @@ use crate::{ parser::ast::{Expression, JoinedStr}, }; pub fn is_at_compound_statement(token: &Token) -> bool { - let kind_is_statement = match token.kind { + match token.kind { Kind::If | Kind::While | Kind::For @@ -19,23 +19,16 @@ pub fn is_at_compound_statement(token: &Token) -> bool { | Kind::With | Kind::Def | Kind::Class + | Kind::Type + | Kind::Match // Decorator | Kind::MatrixMul | Kind::Async => true, _ => false, - }; - if kind_is_statement { - return true; - } - - // Match is a soft keyword so it's an identifier token - if Kind::Identifier == token.kind && token.value.to_string() == "match" { - return true; } - - false } +// TODO: performance pub fn extract_string_inside(val: String) -> String { let delimiters = vec!["\"\"\"", "\"", "'''", "'"]; let mut result = String::new(); diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 4fd12bf3..36bfa840 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -14,7 +14,7 @@ use crate::{ get_row_col_position, lexer::Lexer, parser::{ast::*, extract_string_inside}, - token::{Kind, Token, TokenValue}, + token::{Kind, Token}, }; #[derive(Debug, Clone)] @@ -31,12 +31,11 @@ pub struct Parser<'a> { // This is incremented when we see an opening bracket and decremented when we // see a closing bracket. nested_expression_list: u32, - path: &'a str, } #[allow(unused)] impl<'a> Parser<'a> { - pub fn new(source: &'a str, path: &'a str) -> Self { + pub fn new(source: &'a str) -> Self { let mut lexer = Lexer::new(source); let cur_token = lexer.next_token(); @@ -46,11 +45,12 @@ impl<'a> Parser<'a> { Kind::RightParen | Kind::RightBrace | Kind::RightBracket => nested_expression_list -= 1, _ => {} } - let identifiers_offset = if cur_token.kind == Kind::Identifier { - vec![(cur_token.start, cur_token.end, cur_token.value.to_string())] - } else { - vec![] - }; + let identifiers_offset = + if matches!(cur_token.kind, Kind::Identifier | Kind::Match | Kind::Type) { + vec![(cur_token.start, cur_token.end, cur_token.to_string(source))] + } else { + vec![] + }; let prev_token_end = 0; Self { @@ -60,7 +60,6 @@ impl<'a> Parser<'a> { prev_token_end, prev_nonwhitespace_token_end: prev_token_end, nested_expression_list, - path, identifiers_start_offset: identifiers_offset, } } @@ -126,6 +125,9 @@ impl<'a> Parser<'a> { /// Checks if the current index has token `Kind` fn at(&self, kind: Kind) -> bool { + if kind == Kind::Identifier { + return self.cur_token().can_be_identifier(); + } self.cur_kind() == kind } @@ -154,9 +156,12 @@ impl<'a> Parser<'a> { /// count_in_offset: if true, the offset will be increased by the length of the current token so prev_token_end is adjusted fn advance(&mut self, count_in_offset: bool) { let token = self.lexer.next_token(); - if token.kind == Kind::Identifier { - self.identifiers_start_offset - .push((token.start, token.end, token.value.to_string())); + if matches!(token.kind, Kind::Identifier | Kind::Match | Kind::Type) { + self.identifiers_start_offset.push(( + token.start, + token.end, + token.to_string(self.source), + )); } match token.kind { @@ -191,12 +196,8 @@ impl<'a> Parser<'a> { pub fn expect(&mut self, kind: Kind) -> Result<(), ParsingError> { if !self.at(kind) { let found = &self.cur_token; - let file = &self.path; let line = &self.get_offset_line_number(found.start); - panic!( - "Expected {:?} but found {:?} {file} line: {line:}", - kind, found - ); + panic!("Expected {:?} but found {:?} line: {line:}", kind, found); } self.bump_any(); Ok(()) @@ -216,11 +217,6 @@ impl<'a> Parser<'a> { Ok(()) } - fn unexpected_token_new(&mut self, node: Node, kinds: Vec, advice: &str) -> ParsingError { - let token = self.cur_token(); - panic!("Unexpected token {token:?}"); - } - fn get_offset_line_number(&self, pos: u32) -> u32 { match self.lexer.line_starts.binary_search(&pos) { Ok(line) => line as u32 + 1, @@ -245,33 +241,9 @@ impl<'a> Parser<'a> { Kind::From => self.parse_from_import_statement(), Kind::Global => self.parse_global_statement(), Kind::Nonlocal => self.parse_nonlocal_statement(), - _ => { - if self.cur_kind() == Kind::Identifier - && self.cur_token().value == TokenValue::Type - && self.peek_kind()? == Kind::Identifier - { - self.parse_type_alias_statement() - } else if self.cur_kind() == Kind::Indent { - let node = self.start_node(); - let kind = self.cur_kind(); - return Err(self.unexpected_token_new( - node, - vec![ - Kind::Assert, - Kind::Pass, - Kind::Del, - Kind::Return, - Kind::Yield, - Kind::Raise, - Kind::Break, - Kind::Continue, - Kind::Import, - Kind::From, - Kind::Global, - Kind::Nonlocal, - ], - "Unexpted indent", - )); + other => { + if self.cur_kind() == Kind::Indent { + panic!("Unexpected token {:?}", self.cur_token()); } else { self.parse_assignment_or_expression_statement() } @@ -284,7 +256,7 @@ impl<'a> Parser<'a> { } fn parse_compound_statement(&mut self) -> Result { - let stmt = match self.cur_kind() { + match self.cur_kind() { Kind::If => self.parse_if_statement(), Kind::While => self.parse_while_statement(), Kind::For => self.parse_for_statement(), @@ -297,11 +269,6 @@ impl<'a> Parser<'a> { } Kind::MatrixMul => self.parse_decorated_function_def_or_class_def(), Kind::Class => self.parse_class_definition(vec![], None), - // match is a soft keyword - // https://docs.python.org/3/reference/lexical_analysis.html#soft-keywords - Kind::Identifier if self.cur_token().value == TokenValue::Match => { - self.parse_match_statement() - } Kind::Async => { if matches!(self.peek_kind(), Ok(Kind::Def)) { let node = self.start_node(); @@ -313,19 +280,22 @@ impl<'a> Parser<'a> { } else if matches!(self.peek_kind(), Ok(Kind::With)) { self.parse_with_statement() } else { - let node = self.start_node(); - let kind = self.cur_kind(); - self.bump_any(); panic!(""); } } - _ => { - let range = self.finish_node(self.start_node()); - panic!("Expected compound statement"); + other => { + // type & match are a soft keywords + // https://docs.python.org/3/reference/lexical_analysis.html#soft-keywords + let peeked = self.peek_kind()?; + if other == Kind::Type && peeked == Kind::Identifier { + self.parse_type_alias_statement() + } else if other == Kind::Match && matches!(peeked, Kind::Identifier | Kind::Mul) { + self.parse_match_statement() + } else { + self.parse_assignment_or_expression_statement() + } } - }; - - stmt + } } fn err_if_statement_not_ending_in_new_line_or_semicolon( @@ -427,7 +397,7 @@ impl<'a> Parser<'a> { let iter_list = self.parse_starred_list(Kind::Colon)?; let iter = match iter_list.len() { 0 => { - return Err(self.unexpected_token_new(node, vec![], "Expected expression")); + panic!("Unexpected token {:?}", self.cur_token()); } 1 => iter_list.into_iter().next().unwrap(), _ => Expression::Tuple(Box::new(Tuple { @@ -581,7 +551,7 @@ impl<'a> Parser<'a> { None }; let name = if self.eat(Kind::As) { - let val = Some(self.cur_token().value.to_string()); + let val = Some(self.cur_token().to_string(self.source)); self.bump(Kind::Identifier); val } else { @@ -608,7 +578,7 @@ impl<'a> Parser<'a> { decorators: Vec, is_async: bool, ) -> Result { - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); self.expect(Kind::Identifier)?; let type_params = if self.at(Kind::LeftBrace) { self.parse_type_parameters()? @@ -686,7 +656,7 @@ impl<'a> Parser<'a> { self.start_node() }; self.expect(Kind::Class)?; - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); self.expect(Kind::Identifier)?; let type_params = if self.at(Kind::LeftBrace) { self.parse_type_parameters()? @@ -823,7 +793,7 @@ impl<'a> Parser<'a> { if self.at(Kind::As) { let node = self.start_node(); - let name = Some(self.cur_token().value.to_string()); + let name = Some(self.cur_token().to_string(self.source)); self.bump(Kind::As); Ok(MatchPattern::MatchAs(Box::new(MatchAs { node: self.finish_node(node), @@ -856,7 +826,7 @@ impl<'a> Parser<'a> { Kind::LeftParen => self.parse_sequence_pattern(), Kind::LeftBrace => self.parse_sequence_pattern(), Kind::LeftBracket => self.parse_mapping_pattern(), - Kind::Identifier => { + Kind::Match | Kind::Type | Kind::Identifier => { if matches!(self.peek_kind(), Ok(Kind::Dot)) { let node = self.start_node(); let value = self.parse_attr()?; @@ -892,33 +862,7 @@ impl<'a> Parser<'a> { }, _ => { let node = self.start_node(); - Err(self.unexpected_token_new( - node, - vec![ - Kind::LeftParen, - Kind::LeftBrace, - Kind::LeftBracket, - Kind::Identifier, - Kind::Integer, - Kind::Binary, - Kind::Octal, - Kind::Hexadecimal, - Kind::PointFloat, - Kind::ExponentFloat, - Kind::ImaginaryInteger, - Kind::ImaginaryPointFloat, - Kind::ImaginaryExponentFloat, - Kind::None, - Kind::True, - Kind::False, - Kind::StringLiteral, - Kind::RawBytes, - Kind::Bytes, - Kind::Minus, - Kind::Plus, - ], - "A match pattern starts with these characters", - )) + panic!("Unexpected token {:?}", self.cur_kind()); }, } } @@ -934,7 +878,7 @@ impl<'a> Parser<'a> { } fn parse_capture_or_wildcard_pattern(&mut self) -> Result { - let capture_value = self.cur_token().value.to_string(); + let capture_value = self.cur_token().to_string(self.source); let node = self.start_node(); self.expect(Kind::Identifier)?; // TODO: should also accept as? @@ -972,7 +916,7 @@ impl<'a> Parser<'a> { // in contrast to attribute parsing in primary expression fn parse_attr(&mut self) -> Result { let node = self.start_node(); - let id = self.cur_token.value.take_string(); + let id = self.cur_token.to_string(self.source); let mut expr = Ok(Expression::Name(Box::new(Name { node: self.finish_node(node), id, @@ -980,7 +924,7 @@ impl<'a> Parser<'a> { }))); self.expect(Kind::Identifier); while self.eat(Kind::Dot) { - let attr_val = self.cur_token().value.to_string(); + let attr_val = self.cur_token().to_string(self.source); self.expect(Kind::Identifier)?; expr = Ok(Expression::Attribute(Box::new(Attribute { node: self.finish_node(node), @@ -1011,7 +955,7 @@ impl<'a> Parser<'a> { break; } if self.eat(Kind::Pow) { - rest = Some(self.cur_token().value.to_string()); + rest = Some(self.cur_token().to_string(self.source)); self.bump(Kind::Identifier); // consume the trailing comma self.bump(Kind::Comma); @@ -1041,7 +985,7 @@ impl<'a> Parser<'a> { } fn parse_literal_or_value_pattern(&mut self) -> Result { - if self.cur_kind() == Kind::Identifier && !matches!(self.peek_kind(), Ok(Kind::Colon)) { + if self.at(Kind::Identifier) && !matches!(self.peek_kind(), Ok(Kind::Colon)) { let node = self.start_node(); let value = self.parse_attr()?; self.parse_value_pattern(value, node) @@ -1068,7 +1012,7 @@ impl<'a> Parser<'a> { if self.at(Kind::Identifier) && matches!(self.peek_kind(), Ok(Kind::Assign)) { seen_keyword_pattern = true; - kwd_attrs.push(self.cur_token().value.to_string()); + kwd_attrs.push(self.cur_token().to_string(self.source)); self.bump(Kind::Identifier); self.bump(Kind::Assign); kwd_patterns.push(self.parse_pattern()?); @@ -1102,11 +1046,7 @@ impl<'a> Parser<'a> { self.expect(Kind::RightParen)?; Ok(MatchPattern::MatchSequence(pattern)) } else { - Err(self.unexpected_token_new( - node, - vec![Kind::LeftBrace, Kind::LeftParen], - "Write a sequence pattern here", - )) + panic!("Unexpected token {:?}", self.cur_token()); } } @@ -1399,7 +1339,7 @@ impl<'a> Parser<'a> { self.bump(Kind::Global); let mut names = vec![]; while self.at(Kind::Identifier) { - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); names.push(name); self.bump(Kind::Identifier); if !self.eat(Kind::Comma) { @@ -1418,7 +1358,7 @@ impl<'a> Parser<'a> { self.bump(Kind::Nonlocal); let mut names = vec![]; while self.at(Kind::Identifier) { - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); names.push(name); self.bump(Kind::Identifier); if !self.eat(Kind::Comma) { @@ -1462,7 +1402,7 @@ impl<'a> Parser<'a> { if self.eat(Kind::LeftParen) { while self.at(Kind::Identifier) { let alias_name = self.start_node(); - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); self.bump(Kind::Identifier); let asname = self.parse_alias(name, alias_name); aliases.push(asname); @@ -1474,7 +1414,7 @@ impl<'a> Parser<'a> { } else if self.at(Kind::Identifier) { while self.at(Kind::Identifier) { let alias_name = self.start_node(); - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); self.bump(Kind::Identifier); let asname = self.parse_alias(name, alias_name); aliases.push(asname); @@ -1487,11 +1427,7 @@ impl<'a> Parser<'a> { self.bump_any(); aliases.push(self.parse_alias("*".to_string(), node)); } else { - return Err(self.unexpected_token_new( - import_node, - vec![Kind::Identifier, Kind::Mul, Kind::LeftParen], - "Use * for importing everything or use () to specify names to import or specify the name you want to import" - )); + panic!("Unexpected token {:?}", self.cur_token()); } Ok(Statement::ImportFrom(Box::new(ImportFrom { node: self.finish_node(import_node), @@ -1503,7 +1439,7 @@ impl<'a> Parser<'a> { fn parse_alias(&mut self, name: String, node: Node) -> Alias { let asname = if self.eat(Kind::As) { - let alias_name = self.cur_token().value.to_string(); + let alias_name = self.cur_token().to_string(self.source); self.bump(Kind::Identifier); Some(alias_name) } else { @@ -1534,11 +1470,11 @@ impl<'a> Parser<'a> { } let mut module = String::from(""); if self.at(Kind::Identifier) { - module += &self.cur_token().value.to_string(); + module += &self.cur_token().to_string(self.source); self.bump_any(); while self.eat(Kind::Dot) { module.push('.'); - module.push_str(self.cur_token().value.to_string().as_str()); + module.push_str(self.cur_token().as_str(self.source)); self.expect(Kind::Identifier); } } @@ -1595,7 +1531,7 @@ impl<'a> Parser<'a> { let node = self.start_node(); // TODO: Maybe this walrus check is redundant if self.at(Kind::Identifier) && matches!(self.peek_kind()?, Kind::Walrus) { - let identifier = self.cur_token().value.to_string(); + let identifier = self.cur_token().to_string(self.source); let mut identifier_node = self.start_node(); self.bump_any(); identifier_node = self.finish_node(identifier_node); @@ -1767,7 +1703,7 @@ impl<'a> Parser<'a> { let node = self.start_node(); let mut targets = vec![]; let target = match self.cur_kind() { - Kind::Identifier => match self.peek_kind() { + Kind::Type | Kind::Match | Kind::Identifier => match self.peek_kind() { Ok(Kind::LeftBrace) => { let atom = self.parse_atom()?; self.parse_subscript(node, atom)? @@ -1777,7 +1713,7 @@ impl<'a> Parser<'a> { self.parse_attribute_ref(node, atom)? } _ => { - let identifier = self.cur_token().value.to_string(); + let identifier = self.cur_token().to_string(self.source); let mut identifier_node = self.start_node(); identifier_node = self.finish_node(identifier_node); self.expect(Kind::Identifier)?; @@ -1834,7 +1770,12 @@ impl<'a> Parser<'a> { // check if current kind can be start of a target if !matches!( self.cur_kind(), - Kind::Identifier | Kind::LeftBrace | Kind::LeftParen | Kind::Mul + Kind::Match + | Kind::Type + | Kind::Identifier + | Kind::LeftBrace + | Kind::LeftParen + | Kind::Mul ) { break; } @@ -2276,8 +2217,7 @@ impl<'a> Parser<'a> { } else if self.cur_kind().is_atom() { self.parse_atom()? } else { - let path = &self.path; - panic!("not a primary {} {path:?}", self.cur_token()); + panic!("not a primary {:?}", self.cur_token()); }; let mut primary = if self.at(Kind::Dot) { @@ -2438,7 +2378,7 @@ impl<'a> Parser<'a> { ) -> Result { let mut expr = Ok(value); while self.eat(Kind::Dot) { - let attr_val = self.cur_token().value.to_string(); + let attr_val = self.cur_token().to_string(self.source); self.expect(Kind::Identifier)?; expr = Ok(Expression::Attribute(Box::new(Attribute { node: self.finish_node(node), @@ -2489,8 +2429,8 @@ impl<'a> Parser<'a> { let atom_is_string = self.cur_kind().is_string(); let start = self.start_node(); let mut expr = match self.cur_kind() { - Kind::Identifier => { - let val = self.cur_token.value.take_string(); + Kind::Match | Kind::Type | Kind::Identifier => { + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Name(Box::new(Name { node: self.finish_node(start), @@ -2499,7 +2439,7 @@ impl<'a> Parser<'a> { })) } Kind::Integer => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2528,7 +2468,7 @@ impl<'a> Parser<'a> { })) } Kind::ImaginaryInteger => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2539,10 +2479,9 @@ impl<'a> Parser<'a> { })) } Kind::Bytes => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); let bytes_val = extract_string_inside( - val.to_string() - .strip_prefix('b') + val.strip_prefix('b') .expect("bytes literal must start with b") .to_string(), ) @@ -2554,8 +2493,8 @@ impl<'a> Parser<'a> { })) } Kind::StringLiteral => { - let val = self.cur_token.value.take_string(); - let string_val = extract_string_inside(val.to_string()); + let val = self.cur_token.to_string(self.source); + let string_val = extract_string_inside(val); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2564,11 +2503,10 @@ impl<'a> Parser<'a> { } Kind::RawBytes => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); // rb or br appear in the beginning of raw bytes let bytes_val = - extract_string_inside(val.to_string().chars().skip(2).collect::()) - .into_bytes(); + extract_string_inside(val.chars().skip(2).collect::()).into_bytes(); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2584,7 +2522,7 @@ impl<'a> Parser<'a> { })) } Kind::PointFloat => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2592,7 +2530,7 @@ impl<'a> Parser<'a> { })) } Kind::ExponentFloat => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2600,7 +2538,7 @@ impl<'a> Parser<'a> { })) } Kind::ImaginaryPointFloat => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2611,7 +2549,7 @@ impl<'a> Parser<'a> { })) } Kind::ImaginaryExponentFloat => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2630,7 +2568,7 @@ impl<'a> Parser<'a> { } // TODO: is there something for octal and Hexadecimal? Kind::Octal => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2638,7 +2576,7 @@ impl<'a> Parser<'a> { })) } Kind::Hexadecimal => { - let val = self.cur_token.value.take_string(); + let val = self.cur_token.to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2648,7 +2586,7 @@ impl<'a> Parser<'a> { _ => { let line = self.get_offset_line_number(start.start); let kind = self.cur_kind(); - let value = self.cur_token().value.to_string(); + let value = self.cur_token().to_string(self.source); panic!("token {kind:?} {value:?} {start:?} is not atom"); } }; @@ -2662,8 +2600,8 @@ impl<'a> Parser<'a> { // TODO: duplicate code let start = self.start_node(); let next_str = match self.cur_kind() { - Kind::Identifier => { - let val = self.cur_token().value.to_string(); + Kind::Type | Kind::Match | Kind::Identifier => { + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Name(Box::new(Name { node: self.finish_node(start), @@ -2672,7 +2610,7 @@ impl<'a> Parser<'a> { })) } Kind::Integer => { - let val = self.cur_token().value.to_string(); + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2701,7 +2639,7 @@ impl<'a> Parser<'a> { })) } Kind::ImaginaryInteger => { - let val = self.cur_token().value.to_string(); + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2712,10 +2650,9 @@ impl<'a> Parser<'a> { })) } Kind::Bytes => { - let val = &self.cur_token().value; + let val = &self.cur_token().to_string(self.source); let bytes_val = extract_string_inside( - val.to_string() - .strip_prefix('b') + val.strip_prefix('b') .expect("bytes literal must start with b") .to_string(), ) @@ -2727,8 +2664,8 @@ impl<'a> Parser<'a> { })) } Kind::StringLiteral => { - let val = &self.cur_token().value; - let string_val = extract_string_inside(val.to_string()); + let string_val = + extract_string_inside(self.cur_token().to_string(self.source)); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2737,12 +2674,11 @@ impl<'a> Parser<'a> { } Kind::RawBytes => { - let val = &self.cur_token().value; + let val = &self.cur_token().to_string(self.source); // rb or br appear in the beginning of raw bytes - let bytes_val = extract_string_inside( - val.to_string().chars().skip(2).collect::(), - ) - .into_bytes(); + let bytes_val = + extract_string_inside(val.chars().skip(2).collect::()) + .into_bytes(); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2758,7 +2694,7 @@ impl<'a> Parser<'a> { })) } Kind::PointFloat => { - let val = self.cur_token().value.to_string(); + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2766,7 +2702,7 @@ impl<'a> Parser<'a> { })) } Kind::ExponentFloat => { - let val = self.cur_token().value.to_string(); + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2774,7 +2710,7 @@ impl<'a> Parser<'a> { })) } Kind::ImaginaryPointFloat => { - let val = self.cur_token().value.to_string(); + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2785,7 +2721,7 @@ impl<'a> Parser<'a> { })) } Kind::ImaginaryExponentFloat => { - let val = self.cur_token().value.to_string(); + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2804,7 +2740,7 @@ impl<'a> Parser<'a> { } // TODO: is there something for octal and Hexadecimal? Kind::Octal => { - let val = self.cur_token().value.to_string(); + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2812,7 +2748,7 @@ impl<'a> Parser<'a> { })) } Kind::Hexadecimal => { - let val = self.cur_token().value.to_string(); + let val = self.cur_token().to_string(self.source); self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), @@ -2822,7 +2758,7 @@ impl<'a> Parser<'a> { _ => { let line = self.get_offset_line_number(start.start); let kind = self.cur_kind(); - let value = self.cur_token().value.to_string(); + let value = self.cur_token().to_string(self.source); panic!("token {kind:?} {value:?} {start:?} is not atom"); } }; @@ -2854,7 +2790,7 @@ impl<'a> Parser<'a> { fn parse_identifier(&mut self) -> Result { let node = self.start_node(); - let value = self.cur_token().value.to_string(); + let value = self.cur_token().to_string(self.source); self.expect(Kind::Identifier)?; Ok(Expression::Name(Box::new(Name { node: self.finish_node_chomped(node), @@ -3193,7 +3129,7 @@ impl<'a> Parser<'a> { fn parse_keyword_item(&mut self) -> Result { let node = self.start_node(); - let arg = self.cur_token().value.to_string(); + let arg = self.cur_token().to_string(self.source); self.expect(Kind::Identifier); self.expect(Kind::Assign); let value = self.parse_expression()?; @@ -3219,7 +3155,7 @@ impl<'a> Parser<'a> { let mut defaults = vec![]; loop { - if self.is_def_parameter() { + if self.at(Kind::Identifier) { let (param, default) = self.parse_parameter(is_lambda)?; if seen_vararg { kwonlyargs.push(param); @@ -3289,18 +3225,12 @@ impl<'a> Parser<'a> { }) } - fn is_def_parameter(&mut self) -> bool { - self.cur_kind() == Kind::Identifier - // && matches!(self.peek_kind(), Ok(Kind::Assign)) - // || matches!(self.peek_kind(), Ok(Kind::Colon)) - } - fn parse_parameter( &mut self, is_lambda: bool, ) -> Result<(Arg, Option), ParsingError> { let node = self.start_node(); - let arg = self.cur_token().value.to_string(); + let arg = self.cur_token().to_string(self.source); self.bump(Kind::Identifier); // Lambda parameters cannot have annotations let annotation = if self.at(Kind::Colon) && !is_lambda { @@ -3348,9 +3278,9 @@ impl<'a> Parser<'a> { let mut type_params = vec![]; while !self.eat(Kind::RightBrace) { match self.cur_kind() { - Kind::Identifier => { + Kind::Match | Kind::Type | Kind::Identifier => { let node = self.start_node(); - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); self.bump(Kind::Identifier); let bound = if self.eat(Kind::Colon) { Some(self.parse_expression()?) @@ -3367,7 +3297,7 @@ impl<'a> Parser<'a> { // param spec let node = self.start_node(); self.bump(Kind::Pow); - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); self.bump(Kind::Identifier); type_params.push(TypeParam::ParamSpec(ParamSpec { node: self.finish_node(node), @@ -3378,7 +3308,7 @@ impl<'a> Parser<'a> { // type var tuple let node = self.start_node(); self.bump(Kind::Mul); - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); self.bump(Kind::Identifier); type_params.push(TypeParam::TypeVarTuple(TypeVarTuple { node: self.finish_node(node), @@ -3386,11 +3316,7 @@ impl<'a> Parser<'a> { })); } _ => { - return Err(self.unexpected_token_new( - node, - vec![Kind::Identifier, Kind::Pow, Kind::Mul, Kind::RightBracket], - "", - )); + panic!("Unexpected token {:?}", self.cur_token()); } } if !self.at(Kind::RightBrace) { @@ -3398,11 +3324,7 @@ impl<'a> Parser<'a> { } } if type_params.is_empty() { - return Err(self.unexpected_token_new( - node, - vec![Kind::Identifier, Kind::Pow, Kind::Mul], - "Type parameter is empty", - )); + panic!("Unexpected token {:?}", self.cur_token()); } Ok(type_params) } @@ -3410,7 +3332,7 @@ impl<'a> Parser<'a> { fn parse_type_alias_statement(&mut self) -> std::result::Result { let node = self.start_node(); self.expect(Kind::Identifier)?; - let name = self.cur_token().value.to_string(); + let name = self.cur_token().to_string(self.source); self.expect(Kind::Identifier)?; let type_params = if self.eat(Kind::LeftBrace) { let type_params = self.parse_type_parameters()?; @@ -3444,7 +3366,7 @@ impl<'a> Parser<'a> { conversion = 114; } if self.eat(Kind::Exclamation) { - conversion = match self.cur_token.value.take_string().as_str() { + conversion = match self.cur_token.as_str(self.source) { "s" => 115, "r" => 114, "a" => 97, @@ -3479,7 +3401,7 @@ impl<'a> Parser<'a> { let node = self.start_node(); match self.cur_kind() { Kind::FStringMiddle => { - let str_val = self.cur_token().value.to_string(); + let str_val = self.cur_token().to_string(self.source); self.bump(Kind::FStringMiddle); Ok(Expression::Constant(Box::new(Constant { node: self.finish_node(node), @@ -3488,7 +3410,7 @@ impl<'a> Parser<'a> { } Kind::LeftBracket => self.parse_fstring_replacement_field(), _ => { - panic!("kind is {}", self.cur_token()); + panic!("kind is {}", self.cur_token().kind); } } } @@ -3541,7 +3463,7 @@ mod tests { "a |= 1", // annotated assignment ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3557,7 +3479,7 @@ mod tests { #[test] fn test_parse_assert_stmt() { for test_case in &["assert a", "assert a, b", "assert True, 'fancy message'"] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3573,7 +3495,7 @@ mod tests { #[test] fn test_pass_stmt() { for test_case in &["pass", "pass ", "pass\n"] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3589,7 +3511,7 @@ mod tests { #[test] fn test_parse_del_stmt() { for test_case in &["del a", "del a, b", "del a, b, "] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3605,7 +3527,7 @@ mod tests { #[test] fn parse_yield_statement() { for test_case in &["yield", "yield a", "yield a, b", "yield a, b, "] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3621,7 +3543,7 @@ mod tests { #[test] fn test_raise_statement() { for test_case in &["raise", "raise a", "raise a from c"] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3637,7 +3559,7 @@ mod tests { #[test] fn test_parse_break_continue() { for test_case in &["break", "continue"] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3653,7 +3575,7 @@ mod tests { #[test] fn test_parse_bool_op() { for test_case in &["a or b", "a and b", "a or b or c", "a and b or c"] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3669,7 +3591,7 @@ mod tests { #[test] fn test_parse_unary_op() { for test_case in &["not a", "+ a", "~ a", "-a"] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3686,7 +3608,7 @@ mod tests { fn test_named_expression() { { let test_case = &"(a := b)"; - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3714,7 +3636,7 @@ mod tests { )", "(a, b, c,)", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3730,7 +3652,7 @@ mod tests { #[test] fn test_yield_expression() { for test_case in &["yield", "yield a", "yield from a"] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3747,7 +3669,7 @@ mod tests { fn test_starred() { { let test_case = &"(*a)"; - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3764,7 +3686,7 @@ mod tests { fn test_await_expression() { { let test_case = &"await a"; - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3780,7 +3702,7 @@ mod tests { #[test] fn test_attribute_ref() { for test_case in &["a.b", "a.b.c", "a.b_c", "a.b.c.d"] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3806,7 +3728,7 @@ mod tests { "func(a,)", "eval(' '.join(map(chr, [105, 110, 116])))", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3832,7 +3754,7 @@ mod tests { "lambda a=1 : a", "lambda a=1 : a,", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3849,7 +3771,7 @@ mod tests { fn test_conditional_expression() { { let test_case = &"a if b else c if d else e"; - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3881,7 +3803,7 @@ mod tests { "'d' f'a' 'b'", "f'a_{1}' 'b' ", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3904,7 +3826,7 @@ mod tests { // unsupported // "f'hello_{f'''{a}'''}'", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3932,7 +3854,7 @@ mod tests { "a not in b", "a < b < c", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -3957,7 +3879,7 @@ else: b = 1 ", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -4005,7 +3927,7 @@ except *Exception as e: pass ", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -4027,7 +3949,7 @@ except *Exception as e: "a = ...", "... + 1", ] { - let mut parser = Parser::new(test_case, ""); + let mut parser = Parser::new(test_case); let program = parser.parse().expect("parsing failed"); insta::with_settings!({ @@ -4047,7 +3969,6 @@ except *Exception as e: let test_case = fs::read_to_string($test_file).unwrap(); let mut parser = Parser::new( &test_case, - $test_file, ); let program = parser.parse().expect("parsing failed"); let snapshot = format!("{program:#?}"); diff --git a/parser/src/token.rs b/parser/src/token.rs index 3ad6d550..4970d305 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -1,23 +1,56 @@ -use core::panic; use std::fmt::Display; #[derive(Debug, Clone, PartialEq)] pub struct Token { pub kind: Kind, - // Value might be deleted in the future - pub value: TokenValue, pub start: u32, pub end: u32, } -impl Display for Token { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let kind: &str = self.kind.into(); - let value = format!("({:?})", self.value); - let start = self.start.to_string(); - let end = self.end.to_string(); +impl Token { + pub fn as_str<'a>(&self, source: &'a str) -> &'a str { + &source[self.start as usize..self.end as usize] + } + + pub fn to_string(&self, source: &str) -> String { + return self.as_str(source).to_string(); + } + + pub fn display_token(&self, source: &str) -> String { + let kind = self.kind; + let start = self.start; + let end = self.end; + match self.kind { + Kind::Integer + | Kind::Binary + | Kind::Octal + | Kind::Hexadecimal + | Kind::PointFloat + | Kind::ExponentFloat + | Kind::ImaginaryInteger + | Kind::ImaginaryPointFloat + | Kind::ImaginaryExponentFloat + | Kind::StringLiteral + | Kind::FStringMiddle + | Kind::Identifier + | Kind::Bytes => { + let value = self.as_str(source); + if kind == Kind::FStringMiddle { + let new_value = value.replace("{{", "{").replace("}}", "}"); + format!("{},{}: {} {}", start, end, kind, new_value) + } else { + format!("{},{}: {} {}", start, end, kind, value) + } + } + + _ => { + format!("{},{}: {}", start, end, kind) + } + } + } - write!(f, "{},{}: {} {}", start, end, kind, value,) + pub fn can_be_identifier(&self) -> bool { + matches!(self.kind, Kind::Identifier | Kind::Type | Kind::Match) } } @@ -72,6 +105,10 @@ pub enum Kind { With, // with Yield, // yield + // Soft Keywords + Type, + Match, + // String Literals StringLiteral, // f-string literal @@ -234,6 +271,8 @@ impl Kind { | Kind::LeftBrace | Kind::Yield | Kind::Ellipsis + | Kind::Match + | Kind::Type | Kind::None => true, _ => false, } @@ -367,6 +406,8 @@ impl From for &str { Kind::Dedent => "Dedent", Kind::Ellipsis => "Ellipsis", Kind::Exclamation => "!", + Kind::Match => "match", + Kind::Type => "type", } } } @@ -376,52 +417,3 @@ impl Display for Kind { write!(f, "{}", Into::<&str>::into(*self)) } } - -#[derive(Debug, PartialEq, Clone, is_macro::Is)] -pub enum TokenValue { - None, - Number(String), - Str(String), - Indent(usize), - // Soft keywords are special values - Type, - Match, -} - -impl TokenValue { - pub fn take_string(&mut self) -> String { - match self { - TokenValue::Str(s) => std::mem::take(s), - TokenValue::Number(s) => std::mem::take(s), - _ => { - panic!("not a str") - } - } - } -} - -impl Display for TokenValue { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - TokenValue::None => write!(f, "None"), - TokenValue::Number(n) => write!(f, "{}", n), - TokenValue::Str(s) => write!(f, "{}", s), - TokenValue::Indent(i) => write!(f, "{}", i), - TokenValue::Type => write!(f, "type"), - TokenValue::Match => write!(f, "match"), - } - } -} - -impl From for &str { - fn from(val: TokenValue) -> Self { - match val { - TokenValue::None => "None", - TokenValue::Number(_) => "Number", - TokenValue::Str(_) => "Str", - TokenValue::Indent(_) => "Indent", - TokenValue::Type => "type", - TokenValue::Match => "match", - } - } -} diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-0.snap index 33984594..ecf485c3 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "b\"hello\"" --- -0,8: Bytes (Str("b\"hello\"")) +0,8: Bytes b"hello" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-1.snap index 9b07819c..99d8e0af 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-1.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "b\"world\"" --- -0,8: Bytes (Str("b\"world\"")) +0,8: Bytes b"world" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-2.snap index e599329e..2829d7be 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-2.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "b\"\"" --- -0,3: Bytes (Str("b\"\"")) +0,3: Bytes b"" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-3.snap index 2c8bef9f..59048327 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-3.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "a = b\"hello\"" --- -0,1: Identifier (Str("a")) -2,3: = (None) -4,12: Bytes (Str("b\"hello\"")) +0,1: Identifier a +2,3: = +4,12: Bytes b"hello" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-4.snap index 6bf6ec6c..f40229bb 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-4.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "b'hello'" --- -0,8: Bytes (Str("b'hello'")) +0,8: Bytes b'hello' diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-5.snap index f77871fe..d3e2be28 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-5.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "b\"\"\"hello\"\"\"" --- -0,12: Bytes (Str("b\"\"\"hello\"\"\"")) +0,12: Bytes b"""hello""" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-6.snap index a7311bdf..1422c242 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@bytes-literals-6.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "b'''hello'''" --- -0,12: Bytes (Str("b'''hello'''")) +0,12: Bytes b'''hello''' diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@ellipsis-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@ellipsis-0.snap index 72dfbd5f..7b44ac26 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@ellipsis-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@ellipsis-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "..." --- -0,3: Ellipsis (None) +0,3: Ellipsis diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@ellipsis-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@ellipsis-1.snap index 36281fc8..0d3fe532 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@ellipsis-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@ellipsis-1.snap @@ -2,11 +2,11 @@ source: parser/src/lexer/mod.rs description: "def a():\n ..." --- -0,3: Def (None) -4,5: Identifier (Str("a")) -5,6: ( (None) -6,7: ) (None) -7,8: : (None) -8,9: NewLine (None) -9,13: Indent (Indent(1)) -13,16: Ellipsis (None) +0,3: Def +4,5: Identifier a +5,6: ( +6,7: ) +7,8: : +8,9: NewLine +9,13: Indent +13,16: Ellipsis diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-0.snap index 98d0bc17..ee74142d 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-0.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "f\"hello\"" --- -0,2: FStringStart (Str("f\"")) -2,7: FstringMiddle (Str("hello")) -7,8: FStringEnd (Str("\"")) +0,2: FStringStart +2,7: FstringMiddle hello +7,8: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-1.snap index 04059599..5bc5a711 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-1.snap @@ -2,9 +2,9 @@ source: parser/src/lexer/mod.rs description: "f'hello_{var}'" --- -0,2: FStringStart (Str("f'")) -2,8: FstringMiddle (Str("hello_")) -8,9: { (None) -9,12: Identifier (Str("var")) -12,13: } (None) -13,14: FStringEnd (Str("'")) +0,2: FStringStart +2,8: FstringMiddle hello_ +8,9: { +9,12: Identifier var +12,13: } +13,14: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-10.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-10.snap index fe50f758..08270f9e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-10.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-10.snap @@ -2,13 +2,13 @@ source: parser/src/lexer/mod.rs description: "f'hello_{f'''{a}'''}'" --- -0,2: FStringStart (Str("f'")) -2,8: FstringMiddle (Str("hello_")) -8,9: { (None) -9,13: FStringStart (Str("f'''")) -13,14: { (None) -14,15: Identifier (Str("a")) -15,16: } (None) -16,19: FStringEnd (Str("'''")) -19,20: } (None) -20,21: FStringEnd (Str("'")) +0,2: FStringStart +2,8: FstringMiddle hello_ +8,9: { +9,13: FStringStart +13,14: { +14,15: Identifier a +15,16: } +16,19: FStringEnd +19,20: } +20,21: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-2.snap index e3b2ca89..278b6c1d 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-2.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "f\"world\"" --- -0,2: FStringStart (Str("f\"")) -2,7: FstringMiddle (Str("world")) -7,8: FStringEnd (Str("\"")) +0,2: FStringStart +2,7: FstringMiddle world +7,8: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-3.snap index 5483c1c9..aaea25be 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-3.snap @@ -2,5 +2,5 @@ source: parser/src/lexer/mod.rs description: "f\"\"" --- -0,2: FStringStart (Str("f\"")) -2,3: FStringEnd (Str("\"")) +0,2: FStringStart +2,3: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-4.snap index 7a671fc9..1f4cd678 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-4.snap @@ -2,8 +2,8 @@ source: parser/src/lexer/mod.rs description: "a = f\"hello\"" --- -0,1: Identifier (Str("a")) -2,3: = (None) -4,6: FStringStart (Str("f\"")) -6,11: FstringMiddle (Str("hello")) -11,12: FStringEnd (Str("\"")) +0,1: Identifier a +2,3: = +4,6: FStringStart +6,11: FstringMiddle hello +11,12: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-5.snap index da293e82..aeb48818 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-5.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "f\"\"\"hello\"\"\"" --- -0,4: FStringStart (Str("f\"\"\"")) -4,9: FstringMiddle (Str("hello")) -9,12: FStringEnd (Str("\"\"\"")) +0,4: FStringStart +4,9: FstringMiddle hello +9,12: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-6.snap index 20224281..42b86a8b 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-6.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "f'''hello'''" --- -0,4: FStringStart (Str("f'''")) -4,9: FstringMiddle (Str("hello")) -9,12: FStringEnd (Str("'''")) +0,4: FStringStart +4,9: FstringMiddle hello +9,12: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-7.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-7.snap index ccd1da85..1062a369 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-7.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-7.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "f\"{{hey}}\"" --- -0,2: FStringStart (Str("f\"")) -2,7: FstringMiddle (Str("{hey}")) -9,10: FStringEnd (Str("\"")) +0,2: FStringStart +2,9: FstringMiddle {hey} +9,10: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-8.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-8.snap index 247f0b2b..03f76323 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-8.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-8.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "f\"oh_{{hey}}\"" --- -0,2: FStringStart (Str("f\"")) -2,10: FstringMiddle (Str("oh_{hey}")) -12,13: FStringEnd (Str("\"")) +0,2: FStringStart +2,12: FstringMiddle oh_{hey} +12,13: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-9.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-9.snap index d2935aa0..9b24d092 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-9.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-9.snap @@ -2,7 +2,7 @@ source: parser/src/lexer/mod.rs description: "f'a' 'c'" --- -0,2: FStringStart (Str("f'")) -2,3: FstringMiddle (Str("a")) -3,4: FStringEnd (Str("'")) -5,8: StringLiteral (Str("'c'")) +0,2: FStringStart +2,3: FstringMiddle a +3,4: FStringEnd +5,8: StringLiteral 'c' diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@fstring_escape-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@fstring_escape-0.snap index 2ba64438..fe79f241 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@fstring_escape-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@fstring_escape-0.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "f'Can\\'t find plugin'" --- -0,2: FStringStart (Str("f'")) -2,20: FstringMiddle (Str("Can\\'t find plugin")) -20,21: FStringEnd (Str("'")) +0,2: FStringStart +2,20: FstringMiddle Can\'t find plugin +20,21: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@fstring_escape-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@fstring_escape-1.snap index 1c18bd4e..d2b8b290 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@fstring_escape-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@fstring_escape-1.snap @@ -2,15 +2,15 @@ source: parser/src/lexer/mod.rs description: "f\"{{{', '.join(dict_items)}}}\"" --- -0,2: FStringStart (Str("f\"")) -2,3: FstringMiddle (Str("{")) -4,5: { (None) -5,9: StringLiteral (Str("', '")) -9,10: . (None) -10,14: Identifier (Str("join")) -14,15: ( (None) -15,25: Identifier (Str("dict_items")) -25,26: ) (None) -26,27: } (None) -27,28: FstringMiddle (Str("}")) -29,30: FStringEnd (Str("\"")) +0,2: FStringStart +2,4: FstringMiddle { +4,5: { +5,9: StringLiteral ', ' +9,10: . +10,14: Identifier join +14,15: ( +15,25: Identifier dict_items +25,26: ) +26,27: } +27,29: FstringMiddle } +29,30: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-0.snap index 0393120f..3ff4f0c2 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: a --- -0,1: Identifier (Str("a")) +0,1: Identifier a diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-1.snap index 38452a32..b259adc6 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-1.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: a_a --- -0,3: Identifier (Str("a_a")) +0,3: Identifier a_a diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-2.snap index 48656e3d..b8981597 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-2.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: _a --- -0,2: Identifier (Str("_a")) +0,2: Identifier _a diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-3.snap index 0fed3290..f022e3ca 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-3.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: a_ --- -0,2: Identifier (Str("a_")) +0,2: Identifier a_ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-4.snap index 75f64d53..2dfab24b 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-4.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: a_a_a --- -0,5: Identifier (Str("a_a_a")) +0,5: Identifier a_a_a diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-5.snap index eaf7c879..6267aa25 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-5.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: a_a_ --- -0,4: Identifier (Str("a_a_")) +0,4: Identifier a_a_ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-6.snap index 40c9cff0..5036f619 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@identifiers-6.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ಠ_ಠ --- -0,7: Identifier (Str("ಠ_ಠ")) +0,7: Identifier ಠ_ಠ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-0.snap index db060e5b..683f663f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-0.snap @@ -2,5 +2,5 @@ source: parser/src/lexer/mod.rs description: import a --- -0,6: Import (None) -7,8: Identifier (Str("a")) +0,6: Import +7,8: Identifier a diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-1.snap index a6c8e7fa..4a2dd387 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-1.snap @@ -2,7 +2,7 @@ source: parser/src/lexer/mod.rs description: import a.b --- -0,6: Import (None) -7,8: Identifier (Str("a")) -8,9: . (None) -9,10: Identifier (Str("b")) +0,6: Import +7,8: Identifier a +8,9: . +9,10: Identifier b diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-2.snap index 80ec8092..d2e31985 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-2.snap @@ -2,9 +2,9 @@ source: parser/src/lexer/mod.rs description: import a.b.c --- -0,6: Import (None) -7,8: Identifier (Str("a")) -8,9: . (None) -9,10: Identifier (Str("b")) -10,11: . (None) -11,12: Identifier (Str("c")) +0,6: Import +7,8: Identifier a +8,9: . +9,10: Identifier b +10,11: . +11,12: Identifier c diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-3.snap index da71b70b..be0ee875 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@import-3.snap @@ -2,7 +2,7 @@ source: parser/src/lexer/mod.rs description: import a from b --- -0,6: Import (None) -7,8: Identifier (Str("a")) -9,13: From (None) -14,15: Identifier (Str("b")) +0,6: Import +7,8: Identifier a +9,13: From +14,15: Identifier b diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-0.snap index cc33a20f..4029346e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-0.snap @@ -2,11 +2,11 @@ source: parser/src/lexer/mod.rs description: "if True:\n pass\n" --- -0,2: If (None) -3,7: True (None) -7,8: : (None) -8,9: NewLine (None) -9,21: Indent (Indent(1)) -21,25: Pass (None) -25,26: NewLine (None) -26,26: Dedent (Indent(1)) +0,2: If +3,7: True +7,8: : +8,9: NewLine +9,21: Indent +21,25: Pass +25,26: NewLine +26,26: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-1.snap index f60ba84a..93e37371 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-1.snap @@ -2,16 +2,16 @@ source: parser/src/lexer/mod.rs description: "if True:\n pass\nelse:\n pass" --- -0,2: If (None) -3,7: True (None) -7,8: : (None) -8,9: NewLine (None) -9,13: Indent (Indent(1)) -13,17: Pass (None) -17,18: NewLine (None) -18,18: Dedent (Indent(1)) -18,22: Else (None) -22,23: : (None) -23,24: NewLine (None) -24,28: Indent (Indent(1)) -28,32: Pass (None) +0,2: If +3,7: True +7,8: : +8,9: NewLine +9,13: Indent +13,17: Pass +17,18: NewLine +18,18: Dedent +18,22: Else +22,23: : +23,24: NewLine +24,28: Indent +28,32: Pass diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-2.snap index a2431cd1..f011c18e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-2.snap @@ -2,18 +2,18 @@ source: parser/src/lexer/mod.rs description: "if True:\n if True:\n pass\ndef" --- -0,2: If (None) -3,7: True (None) -7,8: : (None) -8,9: NewLine (None) -9,13: Indent (Indent(1)) -13,15: If (None) -16,20: True (None) -20,21: : (None) -21,22: NewLine (None) -22,30: Indent (Indent(1)) -30,34: Pass (None) -34,35: NewLine (None) -35,35: Dedent (Indent(1)) -35,35: Dedent (None) -35,38: Def (None) +0,2: If +3,7: True +7,8: : +8,9: NewLine +9,13: Indent +13,15: If +16,20: True +20,21: : +21,22: NewLine +22,30: Indent +30,34: Pass +34,35: NewLine +35,35: Dedent +35,35: Dedent +35,38: Def diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-3.snap index d5280ee6..1d18fac6 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-3.snap @@ -2,22 +2,22 @@ source: parser/src/lexer/mod.rs description: "def f(x):\n y = z\n\n print(y)\n" --- -0,3: Def (None) -4,5: Identifier (Str("f")) -5,6: ( (None) -6,7: Identifier (Str("x")) -7,8: ) (None) -8,9: : (None) -9,10: NewLine (None) -10,14: Indent (Indent(1)) -14,15: Identifier (Str("y")) -16,17: = (None) -18,19: Identifier (Str("z")) -19,20: NewLine (None) -20,21: NL (None) -25,30: Identifier (Str("print")) -30,31: ( (None) -31,32: Identifier (Str("y")) -32,33: ) (None) -33,34: NewLine (None) -34,34: Dedent (Indent(1)) +0,3: Def +4,5: Identifier f +5,6: ( +6,7: Identifier x +7,8: ) +8,9: : +9,10: NewLine +10,14: Indent +14,15: Identifier y +16,17: = +18,19: Identifier z +19,20: NewLine +20,21: NL +25,30: Identifier print +30,31: ( +31,32: Identifier y +32,33: ) +33,34: NewLine +34,34: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap index c31e7ef0..fc280be1 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@indentation-4.snap @@ -2,17 +2,17 @@ source: parser/src/lexer/mod.rs description: "if a:\n\n f = c\n\n # Path: test_local.py\n" --- -0,2: If (None) -3,4: Identifier (Str("a")) -4,5: : (None) -5,6: NewLine (None) -6,7: NL (None) -7,11: Indent (Indent(1)) -11,12: Identifier (Str("f")) -13,14: = (None) -15,16: Identifier (Str("c")) -16,17: NewLine (None) -17,18: NL (None) -22,43: Comment (Str("# Path: test_local.py")) -43,44: NL (None) -44,44: Dedent (Indent(1)) +0,2: If +3,4: Identifier a +4,5: : +5,6: NewLine +6,7: NL +7,11: Indent +11,12: Identifier f +13,14: = +15,16: Identifier c +16,17: NewLine +17,18: NL +22,43: Comment +43,44: NL +44,44: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-0.snap index 413deadf..5f7ef230 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-0.snap @@ -2,11 +2,11 @@ source: parser/src/lexer/mod.rs description: False None True and as assert async await --- -0,5: False (None) -6,10: None (None) -11,15: True (None) -16,19: And (None) -20,22: As (None) -23,29: Assert (None) -30,35: Async (None) -36,41: Await (None) +0,5: False +6,10: None +11,15: True +16,19: And +20,22: As +23,29: Assert +30,35: Async +36,41: Await diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-1.snap index 4af83e0b..29a2a327 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-1.snap @@ -2,11 +2,11 @@ source: parser/src/lexer/mod.rs description: break class continue def del elif else except --- -0,5: Break (None) -6,11: Class (None) -12,20: Continue (None) -21,24: Def (None) -25,28: Del (None) -29,33: Elif (None) -34,38: Else (None) -39,45: Except (None) +0,5: Break +6,11: Class +12,20: Continue +21,24: Def +25,28: Del +29,33: Elif +34,38: Else +39,45: Except diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-2.snap index f35bdc3e..5501e63c 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-2.snap @@ -2,12 +2,12 @@ source: parser/src/lexer/mod.rs description: finally for from global if import in is lambda --- -0,7: Finally (None) -8,11: For (None) -12,16: From (None) -17,23: Global (None) -24,26: If (None) -27,33: Import (None) -34,36: In (None) -37,39: Is (None) -40,46: Lambda (None) +0,7: Finally +8,11: For +12,16: From +17,23: Global +24,26: If +27,33: Import +34,36: In +37,39: Is +40,46: Lambda diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-3.snap index bcc08e3c..aca4f7cc 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@keywords-3.snap @@ -2,13 +2,13 @@ source: parser/src/lexer/mod.rs description: nonlocal not or pass raise return try while with yield --- -0,8: Nonlocal (None) -9,12: Not (None) -13,15: Or (None) -16,20: Pass (None) -21,26: Raise (None) -27,33: Return (None) -34,37: Try (None) -38,43: While (None) -44,48: With (None) -49,54: Yield (None) +0,8: Nonlocal +9,12: Not +13,15: Or +16,20: Pass +21,26: Raise +27,33: Return +34,37: Try +38,43: While +44,48: With +49,54: Yield diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@newline-in-nested-structure-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@newline-in-nested-structure-0.snap index 328b6a19..1c7c8981 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@newline-in-nested-structure-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@newline-in-nested-structure-0.snap @@ -2,9 +2,9 @@ source: parser/src/lexer/mod.rs description: "(a,\n\n)" --- -0,1: ( (None) -1,2: Identifier (Str("a")) -2,3: , (None) -3,4: NL (None) -4,5: NL (None) -5,6: ) (None) +0,1: ( +1,2: Identifier a +2,3: , +3,4: NL +4,5: NL +5,6: ) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-0.snap index 7c87597e..555e0981 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: 0b0 --- -0,3: Binary (Number("0b0")) +0,3: Binary 0b0 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-1.snap index e00f1814..ff52cdc4 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-1.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: 0b1 --- -0,3: Binary (Number("0b1")) +0,3: Binary 0b1 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-2.snap index 35ce905b..eda659dd 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-2.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: 0b10 --- -0,4: Binary (Number("0b10")) +0,4: Binary 0b10 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-3.snap index 2910472b..f58fa70e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-3.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: 0b11 --- -0,4: Binary (Number("0b11")) +0,4: Binary 0b11 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-4.snap index 7f7b76ce..c41892ad 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-4.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: 0b100 --- -0,5: Binary (Number("0b100")) +0,5: Binary 0b100 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-5.snap index 02cdd9d5..325bc50c 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-5.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: 0b101 --- -0,5: Binary (Number("0b101")) +0,5: Binary 0b101 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-6.snap index 603e0361..87184421 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-6.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: 0b110 --- -0,5: Binary (Number("0b110")) +0,5: Binary 0b110 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-7.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-7.snap index 85623f02..34f0bbd5 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-7.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-binary-7.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: 0b111 --- -0,5: Binary (Number("0b111")) +0,5: Binary 0b111 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-floating-exponent-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-floating-exponent-0.snap index 4251cb8d..360d6b0f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-floating-exponent-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-floating-exponent-0.snap @@ -2,8 +2,8 @@ source: parser/src/lexer/mod.rs description: 0e0 0e-1 0e+2 0e+3j 0e+3J --- -0,3: ExponentFloat (Number("0e0")) -4,8: ExponentFloat (Number("0e-1")) -9,13: ExponentFloat (Number("0e+2")) -14,19: ImaginaryExponentFloat (Number("0e+3j")) -20,25: ImaginaryExponentFloat (Number("0e+3J")) +0,3: ExponentFloat 0e0 +4,8: ExponentFloat 0e-1 +9,13: ExponentFloat 0e+2 +14,19: ImaginaryExponentFloat 0e+3j +20,25: ImaginaryExponentFloat 0e+3J diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-floating-point-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-floating-point-0.snap index 36bbfc48..47fe2d00 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-floating-point-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-floating-point-0.snap @@ -2,9 +2,9 @@ source: parser/src/lexer/mod.rs description: 0.0 0.1 00.0 00.1 0.1j 0.01J --- -0,3: PointFloat (Number("0.0")) -4,7: PointFloat (Number("0.1")) -8,12: PointFloat (Number("00.0")) -13,17: PointFloat (Number("00.1")) -18,22: ImaginaryPointFloat (Number("0.1j")) -23,28: ImaginaryPointFloat (Number("0.01J")) +0,3: PointFloat 0.0 +4,7: PointFloat 0.1 +8,12: PointFloat 00.0 +13,17: PointFloat 00.1 +18,22: ImaginaryPointFloat 0.1j +23,28: ImaginaryPointFloat 0.01J diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-0.snap index 6f07bcc1..ce9faeab 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x0" --- -0,3: Hexadecimal (Number("0x0")) +0,3: Hexadecimal 0x0 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-1.snap index f9631864..6e28aa62 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-1.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x1" --- -0,3: Hexadecimal (Number("0x1")) +0,3: Hexadecimal 0x1 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-10.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-10.snap index 4443b769..ede421e7 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-10.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-10.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xa" --- -0,3: Hexadecimal (Number("0xa")) +0,3: Hexadecimal 0xa diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-11.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-11.snap index 333029cd..c3c33302 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-11.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-11.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xb" --- -0,3: Hexadecimal (Number("0xb")) +0,3: Hexadecimal 0xb diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-12.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-12.snap index f8b511f2..b04c1fa3 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-12.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-12.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xc" --- -0,3: Hexadecimal (Number("0xc")) +0,3: Hexadecimal 0xc diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-13.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-13.snap index 908fb028..56e36f6a 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-13.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-13.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xd" --- -0,3: Hexadecimal (Number("0xd")) +0,3: Hexadecimal 0xd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-14.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-14.snap index efecbb57..a6f14e74 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-14.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-14.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xe" --- -0,3: Hexadecimal (Number("0xe")) +0,3: Hexadecimal 0xe diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-15.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-15.snap index 78c2eff6..db3890c1 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-15.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-15.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xf" --- -0,3: Hexadecimal (Number("0xf")) +0,3: Hexadecimal 0xf diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-16.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-16.snap index c2f7d38f..9a1ca87a 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-16.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-16.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xA" --- -0,3: Hexadecimal (Number("0xA")) +0,3: Hexadecimal 0xA diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-17.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-17.snap index ca2127c3..b0ba5580 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-17.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-17.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xB" --- -0,3: Hexadecimal (Number("0xB")) +0,3: Hexadecimal 0xB diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-18.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-18.snap index 58e7882e..7e627f3f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-18.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-18.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xC" --- -0,3: Hexadecimal (Number("0xC")) +0,3: Hexadecimal 0xC diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-19.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-19.snap index 871d6119..dbb35942 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-19.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-19.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xD" --- -0,3: Hexadecimal (Number("0xD")) +0,3: Hexadecimal 0xD diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-2.snap index 38d7514b..ecdc6bea 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-2.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x2" --- -0,3: Hexadecimal (Number("0x2")) +0,3: Hexadecimal 0x2 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-20.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-20.snap index c6f9e5d2..15720939 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-20.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-20.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xE" --- -0,3: Hexadecimal (Number("0xE")) +0,3: Hexadecimal 0xE diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-21.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-21.snap index 64ab7b27..f6c161d4 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-21.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-21.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0xF" --- -0,3: Hexadecimal (Number("0xF")) +0,3: Hexadecimal 0xF diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-3.snap index 6ab17f5d..befae11c 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-3.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x3" --- -0,3: Hexadecimal (Number("0x3")) +0,3: Hexadecimal 0x3 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-4.snap index 313eb448..2afdf8b5 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-4.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x4" --- -0,3: Hexadecimal (Number("0x4")) +0,3: Hexadecimal 0x4 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-5.snap index 4006df93..07f01fb5 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-5.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x5" --- -0,3: Hexadecimal (Number("0x5")) +0,3: Hexadecimal 0x5 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-6.snap index d44c5e95..f055f8f8 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-6.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x6" --- -0,3: Hexadecimal (Number("0x6")) +0,3: Hexadecimal 0x6 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-7.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-7.snap index 2d100d66..1dfaca07 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-7.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-7.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x7" --- -0,3: Hexadecimal (Number("0x7")) +0,3: Hexadecimal 0x7 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-8.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-8.snap index eb906baa..bd820857 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-8.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-8.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x8" --- -0,3: Hexadecimal (Number("0x8")) +0,3: Hexadecimal 0x8 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-9.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-9.snap index f9c10d38..7f31a66f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-9.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-hexadecimal-9.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "0x9" --- -0,3: Hexadecimal (Number("0x9")) +0,3: Hexadecimal 0x9 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-integer-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-integer-0.snap index 9d6db3b9..f2b58d0a 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-integer-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-integer-0.snap @@ -2,7 +2,7 @@ source: parser/src/lexer/mod.rs description: 11 33 1j 1_000_000j --- -0,2: Integer (Number("11")) -3,5: Integer (Number("33")) -6,8: ImaginaryInteger (Number("1j")) -9,19: ImaginaryInteger (Number("1_000_000j")) +0,2: Integer 11 +3,5: Integer 33 +6,8: ImaginaryInteger 1j +9,19: ImaginaryInteger 1_000_000j diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-0.snap index af18fdd3..271f2113 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-0.snap @@ -1,4 +1,4 @@ --- source: parser/src/lexer/mod.rs --- -0,3: Octal (Number("0o0")) +0,3: Octal 0o0 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-1.snap index 02b14ea2..03d15cad 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-1.snap @@ -1,4 +1,4 @@ --- source: parser/src/lexer/mod.rs --- -0,3: Octal (Number("0o1")) +0,3: Octal 0o1 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-2.snap index 28a6d164..475f7509 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-2.snap @@ -1,4 +1,4 @@ --- source: parser/src/lexer/mod.rs --- -0,3: Octal (Number("0o2")) +0,3: Octal 0o2 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-3.snap index 6a73ea4a..f1e09aec 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-3.snap @@ -1,4 +1,4 @@ --- source: parser/src/lexer/mod.rs --- -0,3: Octal (Number("0o3")) +0,3: Octal 0o3 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-4.snap index 67899985..062ed387 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-4.snap @@ -1,4 +1,4 @@ --- source: parser/src/lexer/mod.rs --- -0,3: Octal (Number("0o4")) +0,3: Octal 0o4 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-5.snap index 3005cc12..eef836b4 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-5.snap @@ -1,4 +1,4 @@ --- source: parser/src/lexer/mod.rs --- -0,3: Octal (Number("0o5")) +0,3: Octal 0o5 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-6.snap index 6d42bd31..3ec21b45 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-6.snap @@ -1,4 +1,4 @@ --- source: parser/src/lexer/mod.rs --- -0,3: Octal (Number("0o6")) +0,3: Octal 0o6 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-7.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-7.snap index 488b3e0a..9585add6 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-7.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@numeric-octal-7.snap @@ -1,4 +1,4 @@ --- source: parser/src/lexer/mod.rs --- -0,3: Octal (Number("0o7")) +0,3: Octal 0o7 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-0.snap index 18fd2b65..7ea94434 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "rb\"hello\"" --- -0,9: RawBytes (Str("rb\"hello\"")) +0,9: RawBytes diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-1.snap index 18051f5e..0478c504 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-1.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "rb\"world\"" --- -0,9: RawBytes (Str("rb\"world\"")) +0,9: RawBytes diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-2.snap index dea439ce..4fb750ee 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-2.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "rb\"\"" --- -0,4: RawBytes (Str("rb\"\"")) +0,4: RawBytes diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-3.snap index dcefa797..3fa5c6d9 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-3.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "a = rb\"hello\"" --- -0,1: Identifier (Str("a")) -2,3: = (None) -4,13: RawBytes (Str("rb\"hello\"")) +0,1: Identifier a +2,3: = +4,13: RawBytes diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-4.snap index e5150604..4253bb98 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-4.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "rb'hello'" --- -0,9: RawBytes (Str("rb'hello'")) +0,9: RawBytes diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-5.snap index decc09b7..5f2fad52 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-5.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "rb\"\"\"hello\"\"\"" --- -0,13: RawBytes (Str("rb\"\"\"hello\"\"\"")) +0,13: RawBytes diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-6.snap index 4e90f86f..97c5a71d 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-bytes-literals-6.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "rb'''hello'''" --- -0,13: RawBytes (Str("rb'''hello'''")) +0,13: RawBytes diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-0.snap index 7a473b35..bf95450a 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-0.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "rf\"hello\"" --- -0,3: RawFString (Str("rf\"")) -3,8: FstringMiddle (Str("hello")) -8,9: FStringEnd (Str("\"")) +0,3: RawFString +3,8: FstringMiddle hello +8,9: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-1.snap index fa7c9da5..d657e917 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-1.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "rf\"world\"" --- -0,3: RawFString (Str("rf\"")) -3,8: FstringMiddle (Str("world")) -8,9: FStringEnd (Str("\"")) +0,3: RawFString +3,8: FstringMiddle world +8,9: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-2.snap index bc1a37e0..43538745 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-2.snap @@ -2,5 +2,5 @@ source: parser/src/lexer/mod.rs description: "rf\"\"" --- -0,3: RawFString (Str("rf\"")) -3,4: FStringEnd (Str("\"")) +0,3: RawFString +3,4: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-3.snap index e72a24be..95a33393 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-3.snap @@ -2,8 +2,8 @@ source: parser/src/lexer/mod.rs description: "a = rf\"hello\"" --- -0,1: Identifier (Str("a")) -2,3: = (None) -4,7: RawFString (Str("rf\"")) -7,12: FstringMiddle (Str("hello")) -12,13: FStringEnd (Str("\"")) +0,1: Identifier a +2,3: = +4,7: RawFString +7,12: FstringMiddle hello +12,13: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-4.snap index d06cd98c..46efe575 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-4.snap @@ -2,9 +2,9 @@ source: parser/src/lexer/mod.rs description: "rf'hello_{var}'" --- -0,3: RawFString (Str("rf'")) -3,9: FstringMiddle (Str("hello_")) -9,10: { (None) -10,13: Identifier (Str("var")) -13,14: } (None) -14,15: FStringEnd (Str("'")) +0,3: RawFString +3,9: FstringMiddle hello_ +9,10: { +10,13: Identifier var +13,14: } +14,15: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-5.snap index 5444d77e..a4e8b825 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-5.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "rf\"\"\"hello\"\"\"" --- -0,5: RawFString (Str("rf\"\"\"")) -5,10: FstringMiddle (Str("hello")) -10,13: FStringEnd (Str("\"\"\"")) +0,5: RawFString +5,10: FstringMiddle hello +10,13: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-6.snap index 1ce107a9..db1a69ec 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-f-string-literals-6.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "rf'''hello'''" --- -0,5: RawFString (Str("rf'''")) -5,10: FstringMiddle (Str("hello")) -10,13: FStringEnd (Str("'''")) +0,5: RawFString +5,10: FstringMiddle hello +10,13: FStringEnd diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-0.snap index 559a6101..d6a36f2e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "r\"hello\"" --- -0,8: StringLiteral (Str("r\"hello\"")) +0,8: StringLiteral r"hello" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-1.snap index abd40474..1af44e5f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-1.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "r\"world\"" --- -0,8: StringLiteral (Str("r\"world\"")) +0,8: StringLiteral r"world" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-2.snap index 54ac380b..57ef4cbb 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-2.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "r\"\"" --- -0,3: StringLiteral (Str("r\"\"")) +0,3: StringLiteral r"" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-3.snap index 483481c1..702b5f3e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-3.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "a = r\"hello\"" --- -0,1: Identifier (Str("a")) -2,3: = (None) -4,12: StringLiteral (Str("r\"hello\"")) +0,1: Identifier a +2,3: = +4,12: StringLiteral r"hello" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-4.snap index e9a4f859..9502edee 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-4.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "r'hello'" --- -0,8: StringLiteral (Str("r'hello'")) +0,8: StringLiteral r'hello' diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-5.snap index c4e74482..35bc084f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-5.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "r\"\"\"hello\"\"\"" --- -0,12: StringLiteral (Str("r\"\"\"hello\"\"\"")) +0,12: StringLiteral r"""hello""" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-6.snap index 9536f30c..1d60444b 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@raw-string-literals-6.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "r'''hello'''" --- -0,12: StringLiteral (Str("r'''hello'''")) +0,12: StringLiteral r'''hello''' diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-0.snap index 1a88ad9b..fe22ced6 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "\"hello\" " --- -0,7: StringLiteral (Str("\"hello\"")) +0,7: StringLiteral "hello" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-1.snap index a729dfa8..15c7f238 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-1.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "\"world\"" --- -0,7: StringLiteral (Str("\"world\"")) +0,7: StringLiteral "world" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-2.snap index 2d25ccff..0544ee29 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-2.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "\"\"" --- -0,2: StringLiteral (Str("\"\"")) +0,2: StringLiteral "" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-3.snap index 87ee003c..9fa8f706 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-3.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "a = \"hello\"" --- -0,1: Identifier (Str("a")) -2,3: = (None) -4,11: StringLiteral (Str("\"hello\"")) +0,1: Identifier a +2,3: = +4,11: StringLiteral "hello" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-4.snap index 4ebbe0a9..bf8006b9 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-4.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "'hello'" --- -0,7: StringLiteral (Str("'hello'")) +0,7: StringLiteral 'hello' diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-5.snap index faaf4472..825bc81c 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-5.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "\"\"\"hello\"\"\"" --- -0,11: StringLiteral (Str("\"\"\"hello\"\"\"")) +0,11: StringLiteral """hello""" diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-6.snap index c889bbe1..d6e13795 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-6.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "'''hello'''" --- -0,11: StringLiteral (Str("'''hello'''")) +0,11: StringLiteral '''hello''' diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-7.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-7.snap index 5c0dc3d1..fa7924cf 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-7.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@string-literals-7.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "'\\n ([^ ].*)'" --- -0,16: StringLiteral (Str("'\\n ([^ ].*)'")) +0,16: StringLiteral '\n ([^ ].*)' diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-0.snap index 047334e4..20b852c0 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-0.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: 1+2 --- -0,1: Integer (Number("1")) -1,2: + (None) -2,3: Integer (Number("2")) +0,1: Integer 1 +1,2: + +2,3: Integer 2 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-1.snap index f5fbb3db..b31f3a96 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-1.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: a+b --- -0,1: Identifier (Str("a")) -1,2: + (None) -2,3: Identifier (Str("b")) +0,1: Identifier a +1,2: + +2,3: Identifier b diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-10.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-10.snap index 36d69c52..85cb2e21 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-10.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-10.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "," --- -0,1: , (None) +0,1: , diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-11.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-11.snap index b4419d68..65c475b6 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-11.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-11.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ; --- -0,1: ; (None) +0,1: ; diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-12.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-12.snap index b5505ec9..2adf01b3 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-12.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-12.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "@" --- -0,1: @ (None) +0,1: @ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-13.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-13.snap index 5b3eedfb..a99dc118 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-13.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-13.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "=" --- -0,1: = (None) +0,1: = diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-14.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-14.snap index a66760ad..4d58ae91 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-14.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-14.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "\\" --- -0,1: \ (None) +0,1: \ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-15.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-15.snap index 38c079a1..ae5031d0 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-15.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-15.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "#" --- -0,1: Comment (Str("#")) +0,1: Comment diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-16.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-16.snap index ff230d6d..3c7ac2f3 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-16.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-16.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: $ --- -0,1: $ (None) +0,1: $ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-17.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-17.snap index 8e49b80a..18dcc2ed 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-17.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-17.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "?" --- -0,1: ? (None) +0,1: ? diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-18.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-18.snap index 8cf8a4da..354dd9c0 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-18.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-18.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "`" --- -0,1: ` (None) +0,1: ` diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-19.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-19.snap index b296a24d..42601f48 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-19.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-19.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "->" --- -0,2: -> (None) +0,2: -> diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-2.snap index 36755fc1..0e81317e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-2.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: a + b --- -0,1: Identifier (Str("a")) -2,3: + (None) -4,5: Identifier (Str("b")) +0,1: Identifier a +2,3: + +4,5: Identifier b diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-20.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-20.snap index 455e3140..ec51dd2f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-20.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-20.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: += --- -0,2: += (None) +0,2: += diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-21.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-21.snap index 0633acae..6e4a730d 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-21.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-21.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "-=" --- -0,2: -= (None) +0,2: -= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-22.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-22.snap index 673efd0a..1f0c9d37 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-22.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-22.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "*=" --- -0,2: *= (None) +0,2: *= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-23.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-23.snap index 853c3ac9..e1eac5f5 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-23.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-23.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: /= --- -0,2: /= (None) +0,2: /= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-24.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-24.snap index 665bc58c..b15b5a65 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-24.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-24.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "%=" --- -0,2: %= (None) +0,2: %= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-25.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-25.snap index 3d9a4ce6..31526eee 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-25.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-25.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "@=" --- -0,2: @= (None) +0,2: @= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-26.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-26.snap index 4977a46b..41de1752 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-26.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-26.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "&=" --- -0,2: &= (None) +0,2: &= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-27.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-27.snap index 69367654..8a2502bd 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-27.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-27.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "|=" --- -0,2: |= (None) +0,2: |= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-28.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-28.snap index 354558f4..6dce4e70 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-28.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-28.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ^= --- -0,2: ^= (None) +0,2: ^= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-29.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-29.snap index fc7c18b7..b22f5f11 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-29.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-29.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: //= --- -0,3: //= (None) +0,3: //= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-3.snap index 4591985f..af8222bb 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-3.snap @@ -2,5 +2,5 @@ source: parser/src/lexer/mod.rs description: +=2 --- -0,2: += (None) -2,3: Integer (Number("2")) +0,2: += +2,3: Integer 2 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-30.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-30.snap index e3fe6ef9..27ddcc9b 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-30.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-30.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "<<=" --- -0,3: <<= (None) +0,3: <<= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-31.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-31.snap index 592b8f90..fe458b68 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-31.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-31.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ">>=" --- -0,3: >>= (None) +0,3: >>= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-32.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-32.snap index cb6480fc..2f3a0f10 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-32.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-32.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "**=" --- -0,3: **= (None) +0,3: **= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-33.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-33.snap index adcdee3d..d0d36a87 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-33.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-33.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "**" --- -0,2: ** (None) +0,2: ** diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-34.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-34.snap index ffe75e76..c68a05df 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-34.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-34.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: // --- -0,2: // (None) +0,2: // diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-35.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-35.snap index b1ffcb87..329f1530 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-35.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-35.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "<<" --- -0,2: << (None) +0,2: << diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-36.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-36.snap index 4aa313bf..1c3318c8 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-36.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-36.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ">>" --- -0,2: >> (None) +0,2: >> diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-37.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-37.snap index c7efa9a9..88569e89 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-37.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-37.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: + --- -0,1: + (None) +0,1: + diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-38.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-38.snap index 2acb73b4..011e01ca 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-38.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-38.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "-" --- -0,1: - (None) +0,1: - diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-39.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-39.snap index 012d0d1c..20620865 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-39.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-39.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "*" --- -0,1: * (None) +0,1: * diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-4.snap index b80314cb..413532a1 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-4.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: xX = 2 --- -0,2: Identifier (Str("xX")) -3,4: = (None) -5,6: Integer (Number("2")) +0,2: Identifier xX +3,4: = +5,6: Integer 2 diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-40.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-40.snap index adcdee3d..d0d36a87 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-40.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-40.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "**" --- -0,2: ** (None) +0,2: ** diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-41.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-41.snap index 127e1aab..0452103e 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-41.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-41.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: / --- -0,1: / (None) +0,1: / diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-42.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-42.snap index ffe75e76..c68a05df 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-42.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-42.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: // --- -0,2: // (None) +0,2: // diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-43.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-43.snap index 2ba7bf76..931a3c79 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-43.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-43.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "%" --- -0,1: % (None) +0,1: % diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-44.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-44.snap index b5505ec9..2adf01b3 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-44.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-44.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "@" --- -0,1: @ (None) +0,1: @ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-45.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-45.snap index b1ffcb87..329f1530 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-45.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-45.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "<<" --- -0,2: << (None) +0,2: << diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-46.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-46.snap index 4aa313bf..1c3318c8 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-46.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-46.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ">>" --- -0,2: >> (None) +0,2: >> diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-47.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-47.snap index 7da9d2e0..389b5405 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-47.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-47.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "&" --- -0,1: & (None) +0,1: & diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-48.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-48.snap index e1d7ef64..e51a9b7b 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-48.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-48.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "|" --- -0,1: | (None) +0,1: | diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-49.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-49.snap index 78527c75..a2cb68f6 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-49.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-49.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ^ --- -0,1: ^ (None) +0,1: ^ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-5.snap index b1b898d5..1d94bff0 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-5.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: if else elif --- -0,2: If (None) -3,7: Else (None) -8,12: Elif (None) +0,2: If +3,7: Else +8,12: Elif diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-50.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-50.snap index b4cf0ff2..30bcf964 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-50.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-50.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "~" --- -0,1: ~ (None) +0,1: ~ diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-51.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-51.snap index 435588ce..7d60b253 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-51.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-51.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ":=" --- -0,2: := (None) +0,2: := diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-52.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-52.snap index a2dfc1f8..b4aeb9b8 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-52.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-52.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "<" --- -0,1: < (None) +0,1: < diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-53.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-53.snap index d2c7ff2c..e04919be 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-53.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-53.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ">" --- -0,1: > (None) +0,1: > diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-54.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-54.snap index 8f5afe38..8a57beb8 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-54.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-54.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "<=" --- -0,2: <= (None) +0,2: <= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-55.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-55.snap index 6a3dc489..4acded1d 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-55.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-55.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: ">=" --- -0,2: >= (None) +0,2: >= diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-56.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-56.snap index fff5e362..0b5e36d0 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-56.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-56.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "==" --- -0,2: == (None) +0,2: == diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-57.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-57.snap index 7d262c9a..da97cad4 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-57.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-57.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "!=" --- -0,2: != (None) +0,2: != diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-6.snap index f0008698..4b690b07 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-6.snap @@ -2,5 +2,5 @@ source: parser/src/lexer/mod.rs description: () --- -0,1: ( (None) -1,2: ) (None) +0,1: ( +1,2: ) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-7.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-7.snap index 6f3afcdc..fccce311 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-7.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-7.snap @@ -2,5 +2,5 @@ source: parser/src/lexer/mod.rs description: "[]" --- -0,1: [ (None) -1,2: ] (None) +0,1: [ +1,2: ] diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-8.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-8.snap index 975d4896..3a8c4dfb 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-8.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-8.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "{}:" --- -0,1: { (None) -1,2: } (None) -2,3: : (None) +0,1: { +1,2: } +2,3: : diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-9.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-9.snap index a2d2d895..3ac820b2 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-9.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@test-lexer-9.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "." --- -0,1: . (None) +0,1: . diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-0.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-0.snap index 854b13fc..7f552fe2 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-0.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-0.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "u\"hello\"" --- -0,8: Unicode (Str("u\"hello\"")) +0,8: Unicode diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-1.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-1.snap index 09eae559..05a2d2c1 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-1.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-1.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "u\"world\"" --- -0,8: Unicode (Str("u\"world\"")) +0,8: Unicode diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-2.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-2.snap index 63e10aa8..d19a9938 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-2.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-2.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "u\"\"" --- -0,3: Unicode (Str("u\"\"")) +0,3: Unicode diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-3.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-3.snap index 6e73ac45..d32a0fe7 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-3.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-3.snap @@ -2,6 +2,6 @@ source: parser/src/lexer/mod.rs description: "a = u\"hello\"" --- -0,1: Identifier (Str("a")) -2,3: = (None) -4,12: Unicode (Str("u\"hello\"")) +0,1: Identifier a +2,3: = +4,12: Unicode diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-4.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-4.snap index 01932aa5..2b033564 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-4.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-4.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "u'hello'" --- -0,8: Unicode (Str("u'hello'")) +0,8: Unicode diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-5.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-5.snap index e0637766..a21b4814 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-5.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-5.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "u\"\"\"hello\"\"\"" --- -0,12: Unicode (Str("u\"\"\"hello\"\"\"")) +0,12: Unicode diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-6.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-6.snap index 51c30253..2f78807a 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-6.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@unicode-string-literals-6.snap @@ -2,4 +2,4 @@ source: parser/src/lexer/mod.rs description: "u'''hello'''" --- -0,12: Unicode (Str("u'''hello'''")) +0,12: Unicode diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@annotated_assignment.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@annotated_assignment.py.snap index d168885f..9e000d77 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@annotated_assignment.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@annotated_assignment.py.snap @@ -3,107 +3,107 @@ source: parser/src/lexer/mod.rs description: "a: int = 1\n\nb: tuple[int, int] = 1, 2\n\nc: tuple[str, int] = \"1\", 2, \n\nd: tuple[int, int] = b\n\ne: tuple[int, int]\n\nf: tuple[int, int] = e\n\ng: tuple[int,\n int] = 1, 2\n\n(h): int = 1\n\n(\n i\n): int = 1\n" input_file: parser/test_data/inputs/annotated_assignment.py --- -0,1: Identifier (Str("a")) -1,2: : (None) -3,6: Identifier (Str("int")) -7,8: = (None) -9,10: Integer (Number("1")) -10,11: NewLine (None) -11,12: NL (None) -12,13: Identifier (Str("b")) -13,14: : (None) -15,20: Identifier (Str("tuple")) -20,21: [ (None) -21,24: Identifier (Str("int")) -24,25: , (None) -26,29: Identifier (Str("int")) -29,30: ] (None) -31,32: = (None) -33,34: Integer (Number("1")) -34,35: , (None) -36,37: Integer (Number("2")) -37,38: NewLine (None) -38,39: NL (None) -39,40: Identifier (Str("c")) -40,41: : (None) -42,47: Identifier (Str("tuple")) -47,48: [ (None) -48,51: Identifier (Str("str")) -51,52: , (None) -53,56: Identifier (Str("int")) -56,57: ] (None) -58,59: = (None) -60,63: StringLiteral (Str("\"1\"")) -63,64: , (None) -65,66: Integer (Number("2")) -66,67: , (None) -68,69: NewLine (None) -69,70: NL (None) -70,71: Identifier (Str("d")) -71,72: : (None) -73,78: Identifier (Str("tuple")) -78,79: [ (None) -79,82: Identifier (Str("int")) -82,83: , (None) -84,87: Identifier (Str("int")) -87,88: ] (None) -89,90: = (None) -91,92: Identifier (Str("b")) -92,93: NewLine (None) -93,94: NL (None) -94,95: Identifier (Str("e")) -95,96: : (None) -97,102: Identifier (Str("tuple")) -102,103: [ (None) -103,106: Identifier (Str("int")) -106,107: , (None) -108,111: Identifier (Str("int")) -111,112: ] (None) -112,113: NewLine (None) -113,114: NL (None) -114,115: Identifier (Str("f")) -115,116: : (None) -117,122: Identifier (Str("tuple")) -122,123: [ (None) -123,126: Identifier (Str("int")) -126,127: , (None) -128,131: Identifier (Str("int")) -131,132: ] (None) -133,134: = (None) -135,136: Identifier (Str("e")) -136,137: NewLine (None) -137,138: NL (None) -138,139: Identifier (Str("g")) -139,140: : (None) -141,146: Identifier (Str("tuple")) -146,147: [ (None) -147,150: Identifier (Str("int")) -150,151: , (None) -151,152: NL (None) -161,164: Identifier (Str("int")) -164,165: ] (None) -166,167: = (None) -168,169: Integer (Number("1")) -169,170: , (None) -171,172: Integer (Number("2")) -172,173: NewLine (None) -173,174: NL (None) -174,175: ( (None) -175,176: Identifier (Str("h")) -176,177: ) (None) -177,178: : (None) -179,182: Identifier (Str("int")) -183,184: = (None) -185,186: Integer (Number("1")) -186,187: NewLine (None) -187,188: NL (None) -188,189: ( (None) -189,190: NL (None) -194,195: Identifier (Str("i")) -195,196: NL (None) -196,197: ) (None) -197,198: : (None) -199,202: Identifier (Str("int")) -203,204: = (None) -205,206: Integer (Number("1")) -206,207: NewLine (None) +0,1: Identifier a +1,2: : +3,6: Identifier int +7,8: = +9,10: Integer 1 +10,11: NewLine +11,12: NL +12,13: Identifier b +13,14: : +15,20: Identifier tuple +20,21: [ +21,24: Identifier int +24,25: , +26,29: Identifier int +29,30: ] +31,32: = +33,34: Integer 1 +34,35: , +36,37: Integer 2 +37,38: NewLine +38,39: NL +39,40: Identifier c +40,41: : +42,47: Identifier tuple +47,48: [ +48,51: Identifier str +51,52: , +53,56: Identifier int +56,57: ] +58,59: = +60,63: StringLiteral "1" +63,64: , +65,66: Integer 2 +66,67: , +68,69: NewLine +69,70: NL +70,71: Identifier d +71,72: : +73,78: Identifier tuple +78,79: [ +79,82: Identifier int +82,83: , +84,87: Identifier int +87,88: ] +89,90: = +91,92: Identifier b +92,93: NewLine +93,94: NL +94,95: Identifier e +95,96: : +97,102: Identifier tuple +102,103: [ +103,106: Identifier int +106,107: , +108,111: Identifier int +111,112: ] +112,113: NewLine +113,114: NL +114,115: Identifier f +115,116: : +117,122: Identifier tuple +122,123: [ +123,126: Identifier int +126,127: , +128,131: Identifier int +131,132: ] +133,134: = +135,136: Identifier e +136,137: NewLine +137,138: NL +138,139: Identifier g +139,140: : +141,146: Identifier tuple +146,147: [ +147,150: Identifier int +150,151: , +151,152: NL +161,164: Identifier int +164,165: ] +166,167: = +168,169: Integer 1 +169,170: , +171,172: Integer 2 +172,173: NewLine +173,174: NL +174,175: ( +175,176: Identifier h +176,177: ) +177,178: : +179,182: Identifier int +183,184: = +185,186: Integer 1 +186,187: NewLine +187,188: NL +188,189: ( +189,190: NL +194,195: Identifier i +195,196: NL +196,197: ) +197,198: : +199,202: Identifier int +203,204: = +205,206: Integer 1 +206,207: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap index 5a3ef973..d17340e8 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@binary_op.py.snap @@ -3,74 +3,74 @@ source: parser/src/lexer/mod.rs description: "1 + 2\n\n1 - 2\n\n1 * 2\n\n1 / 2\n\n1 // 2\n\n1 % 2\n\n1 ** 2\n\n1 << 2\n\n1 >> 2\n\n1 & 2\n\n1 ^ 2\n\n1 | 2\n\n1 | 2 | 3\n\n1 @ 2\n" input_file: parser/test_data/inputs/binary_op.py --- -0,1: Integer (Number("1")) -2,3: + (None) -4,5: Integer (Number("2")) -5,6: NewLine (None) -6,7: NL (None) -7,8: Integer (Number("1")) -9,10: - (None) -11,12: Integer (Number("2")) -12,13: NewLine (None) -13,14: NL (None) -14,15: Integer (Number("1")) -16,17: * (None) -18,19: Integer (Number("2")) -19,20: NewLine (None) -20,21: NL (None) -21,22: Integer (Number("1")) -23,24: / (None) -25,26: Integer (Number("2")) -26,27: NewLine (None) -27,28: NL (None) -28,29: Integer (Number("1")) -30,32: // (None) -33,34: Integer (Number("2")) -34,35: NewLine (None) -35,36: NL (None) -36,37: Integer (Number("1")) -38,39: % (None) -40,41: Integer (Number("2")) -41,42: NewLine (None) -42,43: NL (None) -43,44: Integer (Number("1")) -45,47: ** (None) -48,49: Integer (Number("2")) -49,50: NewLine (None) -50,51: NL (None) -51,52: Integer (Number("1")) -53,55: << (None) -56,57: Integer (Number("2")) -57,58: NewLine (None) -58,59: NL (None) -59,60: Integer (Number("1")) -61,63: >> (None) -64,65: Integer (Number("2")) -65,66: NewLine (None) -66,67: NL (None) -67,68: Integer (Number("1")) -69,70: & (None) -71,72: Integer (Number("2")) -72,73: NewLine (None) -73,74: NL (None) -74,75: Integer (Number("1")) -76,77: ^ (None) -78,79: Integer (Number("2")) -79,80: NewLine (None) -80,81: NL (None) -81,82: Integer (Number("1")) -83,84: | (None) -85,86: Integer (Number("2")) -86,87: NewLine (None) -87,88: NL (None) -88,89: Integer (Number("1")) -90,91: | (None) -92,93: Integer (Number("2")) -94,95: | (None) -96,97: Integer (Number("3")) -97,98: NewLine (None) -98,99: NL (None) -99,100: Integer (Number("1")) -101,102: @ (None) -103,104: Integer (Number("2")) -104,105: NewLine (None) +0,1: Integer 1 +2,3: + +4,5: Integer 2 +5,6: NewLine +6,7: NL +7,8: Integer 1 +9,10: - +11,12: Integer 2 +12,13: NewLine +13,14: NL +14,15: Integer 1 +16,17: * +18,19: Integer 2 +19,20: NewLine +20,21: NL +21,22: Integer 1 +23,24: / +25,26: Integer 2 +26,27: NewLine +27,28: NL +28,29: Integer 1 +30,32: // +33,34: Integer 2 +34,35: NewLine +35,36: NL +36,37: Integer 1 +38,39: % +40,41: Integer 2 +41,42: NewLine +42,43: NL +43,44: Integer 1 +45,47: ** +48,49: Integer 2 +49,50: NewLine +50,51: NL +51,52: Integer 1 +53,55: << +56,57: Integer 2 +57,58: NewLine +58,59: NL +59,60: Integer 1 +61,63: >> +64,65: Integer 2 +65,66: NewLine +66,67: NL +67,68: Integer 1 +69,70: & +71,72: Integer 2 +72,73: NewLine +73,74: NL +74,75: Integer 1 +76,77: ^ +78,79: Integer 2 +79,80: NewLine +80,81: NL +81,82: Integer 1 +83,84: | +85,86: Integer 2 +86,87: NewLine +87,88: NL +88,89: Integer 1 +90,91: | +92,93: Integer 2 +94,95: | +96,97: Integer 3 +97,98: NewLine +98,99: NL +99,100: Integer 1 +101,102: @ +103,104: Integer 2 +104,105: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@class.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@class.py.snap index 08b177c3..1983ffd2 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@class.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@class.py.snap @@ -3,255 +3,255 @@ source: parser/src/lexer/mod.rs description: "class a: pass\n\nclass a():\n pass\n\nclass a(b, c): pass\n\nclass a(b, *c, **d): pass\n\nclass a(b,\nc,\nd): pass\n\n@decor\nclass a: pass\n\nclass a[T]: pass\n\nclass a[T, U]: pass\n\nclass a[T: U]: pass\n\nclass a[T: U, V: W]: pass\n\nclass a[*T]: pass\n\nclass a[T, *U]: pass\n\nclass a[T: U, *V]: pass\n\nclass a[T: U, V: W, *X]: pass\n\nclass a[**T]: pass\n\nclass a[T, **U]: pass\n\nclass a[T: U, **V]: pass\n\nclass a[T: U, V: W, **X]: pass\n\nclass a[T, *U, **V]: pass\n\nclass a[T: U, *V, **W]: pass\n" input_file: parser/test_data/inputs/class.py --- -0,5: Class (None) -6,7: Identifier (Str("a")) -7,8: : (None) -9,13: Pass (None) -13,14: NewLine (None) -14,15: NL (None) -15,20: Class (None) -21,22: Identifier (Str("a")) -22,23: ( (None) -23,24: ) (None) -24,25: : (None) -25,26: NewLine (None) -26,30: Indent (Indent(1)) -30,34: Pass (None) -34,35: NewLine (None) -35,36: NL (None) -36,36: Dedent (Indent(1)) -36,41: Class (None) -42,43: Identifier (Str("a")) -43,44: ( (None) -44,45: Identifier (Str("b")) -45,46: , (None) -47,48: Identifier (Str("c")) -48,49: ) (None) -49,50: : (None) -51,55: Pass (None) -55,56: NewLine (None) -56,57: NL (None) -57,62: Class (None) -63,64: Identifier (Str("a")) -64,65: ( (None) -65,66: Identifier (Str("b")) -66,67: , (None) -68,69: * (None) -69,70: Identifier (Str("c")) -70,71: , (None) -72,74: ** (None) -74,75: Identifier (Str("d")) -75,76: ) (None) -76,77: : (None) -78,82: Pass (None) -82,83: NewLine (None) -83,84: NL (None) -84,89: Class (None) -90,91: Identifier (Str("a")) -91,92: ( (None) -92,93: Identifier (Str("b")) -93,94: , (None) -94,95: NL (None) -95,96: Identifier (Str("c")) -96,97: , (None) -97,98: NL (None) -98,99: Identifier (Str("d")) -99,100: ) (None) -100,101: : (None) -102,106: Pass (None) -106,107: NewLine (None) -107,108: NL (None) -108,109: @ (None) -109,114: Identifier (Str("decor")) -114,115: NewLine (None) -115,120: Class (None) -121,122: Identifier (Str("a")) -122,123: : (None) -124,128: Pass (None) -128,129: NewLine (None) -129,130: NL (None) -130,135: Class (None) -136,137: Identifier (Str("a")) -137,138: [ (None) -138,139: Identifier (Str("T")) -139,140: ] (None) -140,141: : (None) -142,146: Pass (None) -146,147: NewLine (None) -147,148: NL (None) -148,153: Class (None) -154,155: Identifier (Str("a")) -155,156: [ (None) -156,157: Identifier (Str("T")) -157,158: , (None) -159,160: Identifier (Str("U")) -160,161: ] (None) -161,162: : (None) -163,167: Pass (None) -167,168: NewLine (None) -168,169: NL (None) -169,174: Class (None) -175,176: Identifier (Str("a")) -176,177: [ (None) -177,178: Identifier (Str("T")) -178,179: : (None) -180,181: Identifier (Str("U")) -181,182: ] (None) -182,183: : (None) -184,188: Pass (None) -188,189: NewLine (None) -189,190: NL (None) -190,195: Class (None) -196,197: Identifier (Str("a")) -197,198: [ (None) -198,199: Identifier (Str("T")) -199,200: : (None) -201,202: Identifier (Str("U")) -202,203: , (None) -204,205: Identifier (Str("V")) -205,206: : (None) -207,208: Identifier (Str("W")) -208,209: ] (None) -209,210: : (None) -211,215: Pass (None) -215,216: NewLine (None) -216,217: NL (None) -217,222: Class (None) -223,224: Identifier (Str("a")) -224,225: [ (None) -225,226: * (None) -226,227: Identifier (Str("T")) -227,228: ] (None) -228,229: : (None) -230,234: Pass (None) -234,235: NewLine (None) -235,236: NL (None) -236,241: Class (None) -242,243: Identifier (Str("a")) -243,244: [ (None) -244,245: Identifier (Str("T")) -245,246: , (None) -247,248: * (None) -248,249: Identifier (Str("U")) -249,250: ] (None) -250,251: : (None) -252,256: Pass (None) -256,257: NewLine (None) -257,258: NL (None) -258,263: Class (None) -264,265: Identifier (Str("a")) -265,266: [ (None) -266,267: Identifier (Str("T")) -267,268: : (None) -269,270: Identifier (Str("U")) -270,271: , (None) -272,273: * (None) -273,274: Identifier (Str("V")) -274,275: ] (None) -275,276: : (None) -277,281: Pass (None) -281,282: NewLine (None) -282,283: NL (None) -283,288: Class (None) -289,290: Identifier (Str("a")) -290,291: [ (None) -291,292: Identifier (Str("T")) -292,293: : (None) -294,295: Identifier (Str("U")) -295,296: , (None) -297,298: Identifier (Str("V")) -298,299: : (None) -300,301: Identifier (Str("W")) -301,302: , (None) -303,304: * (None) -304,305: Identifier (Str("X")) -305,306: ] (None) -306,307: : (None) -308,312: Pass (None) -312,313: NewLine (None) -313,314: NL (None) -314,319: Class (None) -320,321: Identifier (Str("a")) -321,322: [ (None) -322,324: ** (None) -324,325: Identifier (Str("T")) -325,326: ] (None) -326,327: : (None) -328,332: Pass (None) -332,333: NewLine (None) -333,334: NL (None) -334,339: Class (None) -340,341: Identifier (Str("a")) -341,342: [ (None) -342,343: Identifier (Str("T")) -343,344: , (None) -345,347: ** (None) -347,348: Identifier (Str("U")) -348,349: ] (None) -349,350: : (None) -351,355: Pass (None) -355,356: NewLine (None) -356,357: NL (None) -357,362: Class (None) -363,364: Identifier (Str("a")) -364,365: [ (None) -365,366: Identifier (Str("T")) -366,367: : (None) -368,369: Identifier (Str("U")) -369,370: , (None) -371,373: ** (None) -373,374: Identifier (Str("V")) -374,375: ] (None) -375,376: : (None) -377,381: Pass (None) -381,382: NewLine (None) -382,383: NL (None) -383,388: Class (None) -389,390: Identifier (Str("a")) -390,391: [ (None) -391,392: Identifier (Str("T")) -392,393: : (None) -394,395: Identifier (Str("U")) -395,396: , (None) -397,398: Identifier (Str("V")) -398,399: : (None) -400,401: Identifier (Str("W")) -401,402: , (None) -403,405: ** (None) -405,406: Identifier (Str("X")) -406,407: ] (None) -407,408: : (None) -409,413: Pass (None) -413,414: NewLine (None) -414,415: NL (None) -415,420: Class (None) -421,422: Identifier (Str("a")) -422,423: [ (None) -423,424: Identifier (Str("T")) -424,425: , (None) -426,427: * (None) -427,428: Identifier (Str("U")) -428,429: , (None) -430,432: ** (None) -432,433: Identifier (Str("V")) -433,434: ] (None) -434,435: : (None) -436,440: Pass (None) -440,441: NewLine (None) -441,442: NL (None) -442,447: Class (None) -448,449: Identifier (Str("a")) -449,450: [ (None) -450,451: Identifier (Str("T")) -451,452: : (None) -453,454: Identifier (Str("U")) -454,455: , (None) -456,457: * (None) -457,458: Identifier (Str("V")) -458,459: , (None) -460,462: ** (None) -462,463: Identifier (Str("W")) -463,464: ] (None) -464,465: : (None) -466,470: Pass (None) -470,471: NewLine (None) +0,5: Class +6,7: Identifier a +7,8: : +9,13: Pass +13,14: NewLine +14,15: NL +15,20: Class +21,22: Identifier a +22,23: ( +23,24: ) +24,25: : +25,26: NewLine +26,30: Indent +30,34: Pass +34,35: NewLine +35,36: NL +36,36: Dedent +36,41: Class +42,43: Identifier a +43,44: ( +44,45: Identifier b +45,46: , +47,48: Identifier c +48,49: ) +49,50: : +51,55: Pass +55,56: NewLine +56,57: NL +57,62: Class +63,64: Identifier a +64,65: ( +65,66: Identifier b +66,67: , +68,69: * +69,70: Identifier c +70,71: , +72,74: ** +74,75: Identifier d +75,76: ) +76,77: : +78,82: Pass +82,83: NewLine +83,84: NL +84,89: Class +90,91: Identifier a +91,92: ( +92,93: Identifier b +93,94: , +94,95: NL +95,96: Identifier c +96,97: , +97,98: NL +98,99: Identifier d +99,100: ) +100,101: : +102,106: Pass +106,107: NewLine +107,108: NL +108,109: @ +109,114: Identifier decor +114,115: NewLine +115,120: Class +121,122: Identifier a +122,123: : +124,128: Pass +128,129: NewLine +129,130: NL +130,135: Class +136,137: Identifier a +137,138: [ +138,139: Identifier T +139,140: ] +140,141: : +142,146: Pass +146,147: NewLine +147,148: NL +148,153: Class +154,155: Identifier a +155,156: [ +156,157: Identifier T +157,158: , +159,160: Identifier U +160,161: ] +161,162: : +163,167: Pass +167,168: NewLine +168,169: NL +169,174: Class +175,176: Identifier a +176,177: [ +177,178: Identifier T +178,179: : +180,181: Identifier U +181,182: ] +182,183: : +184,188: Pass +188,189: NewLine +189,190: NL +190,195: Class +196,197: Identifier a +197,198: [ +198,199: Identifier T +199,200: : +201,202: Identifier U +202,203: , +204,205: Identifier V +205,206: : +207,208: Identifier W +208,209: ] +209,210: : +211,215: Pass +215,216: NewLine +216,217: NL +217,222: Class +223,224: Identifier a +224,225: [ +225,226: * +226,227: Identifier T +227,228: ] +228,229: : +230,234: Pass +234,235: NewLine +235,236: NL +236,241: Class +242,243: Identifier a +243,244: [ +244,245: Identifier T +245,246: , +247,248: * +248,249: Identifier U +249,250: ] +250,251: : +252,256: Pass +256,257: NewLine +257,258: NL +258,263: Class +264,265: Identifier a +265,266: [ +266,267: Identifier T +267,268: : +269,270: Identifier U +270,271: , +272,273: * +273,274: Identifier V +274,275: ] +275,276: : +277,281: Pass +281,282: NewLine +282,283: NL +283,288: Class +289,290: Identifier a +290,291: [ +291,292: Identifier T +292,293: : +294,295: Identifier U +295,296: , +297,298: Identifier V +298,299: : +300,301: Identifier W +301,302: , +303,304: * +304,305: Identifier X +305,306: ] +306,307: : +308,312: Pass +312,313: NewLine +313,314: NL +314,319: Class +320,321: Identifier a +321,322: [ +322,324: ** +324,325: Identifier T +325,326: ] +326,327: : +328,332: Pass +332,333: NewLine +333,334: NL +334,339: Class +340,341: Identifier a +341,342: [ +342,343: Identifier T +343,344: , +345,347: ** +347,348: Identifier U +348,349: ] +349,350: : +351,355: Pass +355,356: NewLine +356,357: NL +357,362: Class +363,364: Identifier a +364,365: [ +365,366: Identifier T +366,367: : +368,369: Identifier U +369,370: , +371,373: ** +373,374: Identifier V +374,375: ] +375,376: : +377,381: Pass +381,382: NewLine +382,383: NL +383,388: Class +389,390: Identifier a +390,391: [ +391,392: Identifier T +392,393: : +394,395: Identifier U +395,396: , +397,398: Identifier V +398,399: : +400,401: Identifier W +401,402: , +403,405: ** +405,406: Identifier X +406,407: ] +407,408: : +409,413: Pass +413,414: NewLine +414,415: NL +415,420: Class +421,422: Identifier a +422,423: [ +423,424: Identifier T +424,425: , +426,427: * +427,428: Identifier U +428,429: , +430,432: ** +432,433: Identifier V +433,434: ] +434,435: : +436,440: Pass +440,441: NewLine +441,442: NL +442,447: Class +448,449: Identifier a +449,450: [ +450,451: Identifier T +451,452: : +453,454: Identifier U +454,455: , +456,457: * +457,458: Identifier V +458,459: , +460,462: ** +462,463: Identifier W +463,464: ] +464,465: : +466,470: Pass +470,471: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap index 27517a5b..4dd39d50 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@comments.py.snap @@ -3,37 +3,37 @@ source: parser/src/lexer/mod.rs description: "# a\n# a\n\n# a\n# b\ndef a(): ... # a\n\ndef b():\n # a\n ... # a\n\n# this is a comment only line\n" input_file: parser/test_data/inputs/comments.py --- -0,3: Comment (Str("# a")) -3,4: NL (None) -4,7: Comment (Str("# a")) -7,8: NL (None) -8,9: NL (None) -9,12: Comment (Str("# a")) -12,13: NL (None) -13,16: Comment (Str("# b")) -16,17: NL (None) -17,20: Def (None) -21,22: Identifier (Str("a")) -22,23: ( (None) -23,24: ) (None) -24,25: : (None) -26,29: Ellipsis (None) -30,33: Comment (Str("# a")) -33,34: NewLine (None) -34,35: NL (None) -35,38: Def (None) -39,40: Identifier (Str("b")) -40,41: ( (None) -41,42: ) (None) -42,43: : (None) -43,44: NewLine (None) -48,51: Comment (Str("# a")) -51,52: NL (None) -52,56: Indent (Indent(1)) -56,59: Ellipsis (None) -60,63: Comment (Str("# a")) -63,64: NewLine (None) -64,65: NL (None) -65,94: Comment (Str("# this is a comment only line")) -94,95: NL (None) -95,95: Dedent (Indent(1)) +0,3: Comment +3,4: NL +4,7: Comment +7,8: NL +8,9: NL +9,12: Comment +12,13: NL +13,16: Comment +16,17: NL +17,20: Def +21,22: Identifier a +22,23: ( +23,24: ) +24,25: : +26,29: Ellipsis +30,33: Comment +33,34: NewLine +34,35: NL +35,38: Def +39,40: Identifier b +40,41: ( +41,42: ) +42,43: : +43,44: NewLine +48,51: Comment +51,52: NL +52,56: Indent +56,59: Ellipsis +60,63: Comment +63,64: NewLine +64,65: NL +65,94: Comment +94,95: NL +95,95: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@dict.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@dict.py.snap index 079e315f..7b1b68da 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@dict.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@dict.py.snap @@ -3,162 +3,162 @@ source: parser/src/lexer/mod.rs description: "{a: b, c: d}\n\n{a: b,\nc: d},\n\n{a: b\n, c: d},\n\n{a: b,\nc: d,\n e: f},\n\n{a: b,\n},\n\n{a: b, c: d,}\n\n{1: \"name\" for name in get_names()}\n\n{1: \"name\" for name in get_names()\n}\n\naggregated_key_stats[idx_stat] = {\n **filter_value_NONE,\n **filter_key_found,\n **client_types,\n **market_watch,\n \"symbol\": symbol,\n \"name\": name,\n}\n\ndi = {1: \"name\", **{2: \"name2\"}, 3: \"name3\"}\n" input_file: parser/test_data/inputs/dict.py --- -0,1: { (None) -1,2: Identifier (Str("a")) -2,3: : (None) -4,5: Identifier (Str("b")) -5,6: , (None) -7,8: Identifier (Str("c")) -8,9: : (None) -10,11: Identifier (Str("d")) -11,12: } (None) -12,13: NewLine (None) -13,14: NL (None) -14,15: { (None) -15,16: Identifier (Str("a")) -16,17: : (None) -18,19: Identifier (Str("b")) -19,20: , (None) -20,21: NL (None) -21,22: Identifier (Str("c")) -22,23: : (None) -24,25: Identifier (Str("d")) -25,26: } (None) -26,27: , (None) -27,28: NewLine (None) -28,29: NL (None) -29,30: { (None) -30,31: Identifier (Str("a")) -31,32: : (None) -33,34: Identifier (Str("b")) -34,35: NL (None) -35,36: , (None) -37,38: Identifier (Str("c")) -38,39: : (None) -40,41: Identifier (Str("d")) -41,42: } (None) -42,43: , (None) -43,44: NewLine (None) -44,45: NL (None) -45,46: { (None) -46,47: Identifier (Str("a")) -47,48: : (None) -49,50: Identifier (Str("b")) -50,51: , (None) -51,52: NL (None) -52,53: Identifier (Str("c")) -53,54: : (None) -55,56: Identifier (Str("d")) -56,57: , (None) -57,58: NL (None) -62,63: Identifier (Str("e")) -63,64: : (None) -65,66: Identifier (Str("f")) -66,67: } (None) -67,68: , (None) -68,69: NewLine (None) -69,70: NL (None) -70,71: { (None) -71,72: Identifier (Str("a")) -72,73: : (None) -74,75: Identifier (Str("b")) -75,76: , (None) -76,77: NL (None) -77,78: } (None) -78,79: , (None) -79,80: NewLine (None) -80,81: NL (None) -81,82: { (None) -82,83: Identifier (Str("a")) -83,84: : (None) -85,86: Identifier (Str("b")) -86,87: , (None) -88,89: Identifier (Str("c")) -89,90: : (None) -91,92: Identifier (Str("d")) -92,93: , (None) -93,94: } (None) -94,95: NewLine (None) -95,96: NL (None) -96,97: { (None) -97,98: Integer (Number("1")) -98,99: : (None) -100,106: StringLiteral (Str("\"name\"")) -107,110: For (None) -111,115: Identifier (Str("name")) -116,118: In (None) -119,128: Identifier (Str("get_names")) -128,129: ( (None) -129,130: ) (None) -130,131: } (None) -131,132: NewLine (None) -132,133: NL (None) -133,134: { (None) -134,135: Integer (Number("1")) -135,136: : (None) -137,143: StringLiteral (Str("\"name\"")) -144,147: For (None) -148,152: Identifier (Str("name")) -153,155: In (None) -156,165: Identifier (Str("get_names")) -165,166: ( (None) -166,167: ) (None) -167,168: NL (None) -168,169: } (None) -169,170: NewLine (None) -170,171: NL (None) -171,191: Identifier (Str("aggregated_key_stats")) -191,192: [ (None) -192,200: Identifier (Str("idx_stat")) -200,201: ] (None) -202,203: = (None) -204,205: { (None) -205,206: NL (None) -210,212: ** (None) -212,229: Identifier (Str("filter_value_NONE")) -229,230: , (None) -230,231: NL (None) -235,237: ** (None) -237,253: Identifier (Str("filter_key_found")) -253,254: , (None) -254,255: NL (None) -259,261: ** (None) -261,273: Identifier (Str("client_types")) -273,274: , (None) -274,275: NL (None) -279,281: ** (None) -281,293: Identifier (Str("market_watch")) -293,294: , (None) -294,295: NL (None) -299,307: StringLiteral (Str("\"symbol\"")) -307,308: : (None) -309,315: Identifier (Str("symbol")) -315,316: , (None) -316,317: NL (None) -321,327: StringLiteral (Str("\"name\"")) -327,328: : (None) -329,333: Identifier (Str("name")) -333,334: , (None) -334,335: NL (None) -335,336: } (None) -336,337: NewLine (None) -337,338: NL (None) -338,340: Identifier (Str("di")) -341,342: = (None) -343,344: { (None) -344,345: Integer (Number("1")) -345,346: : (None) -347,353: StringLiteral (Str("\"name\"")) -353,354: , (None) -355,357: ** (None) -357,358: { (None) -358,359: Integer (Number("2")) -359,360: : (None) -361,368: StringLiteral (Str("\"name2\"")) -368,369: } (None) -369,370: , (None) -371,372: Integer (Number("3")) -372,373: : (None) -374,381: StringLiteral (Str("\"name3\"")) -381,382: } (None) -382,383: NewLine (None) +0,1: { +1,2: Identifier a +2,3: : +4,5: Identifier b +5,6: , +7,8: Identifier c +8,9: : +10,11: Identifier d +11,12: } +12,13: NewLine +13,14: NL +14,15: { +15,16: Identifier a +16,17: : +18,19: Identifier b +19,20: , +20,21: NL +21,22: Identifier c +22,23: : +24,25: Identifier d +25,26: } +26,27: , +27,28: NewLine +28,29: NL +29,30: { +30,31: Identifier a +31,32: : +33,34: Identifier b +34,35: NL +35,36: , +37,38: Identifier c +38,39: : +40,41: Identifier d +41,42: } +42,43: , +43,44: NewLine +44,45: NL +45,46: { +46,47: Identifier a +47,48: : +49,50: Identifier b +50,51: , +51,52: NL +52,53: Identifier c +53,54: : +55,56: Identifier d +56,57: , +57,58: NL +62,63: Identifier e +63,64: : +65,66: Identifier f +66,67: } +67,68: , +68,69: NewLine +69,70: NL +70,71: { +71,72: Identifier a +72,73: : +74,75: Identifier b +75,76: , +76,77: NL +77,78: } +78,79: , +79,80: NewLine +80,81: NL +81,82: { +82,83: Identifier a +83,84: : +85,86: Identifier b +86,87: , +88,89: Identifier c +89,90: : +91,92: Identifier d +92,93: , +93,94: } +94,95: NewLine +95,96: NL +96,97: { +97,98: Integer 1 +98,99: : +100,106: StringLiteral "name" +107,110: For +111,115: Identifier name +116,118: In +119,128: Identifier get_names +128,129: ( +129,130: ) +130,131: } +131,132: NewLine +132,133: NL +133,134: { +134,135: Integer 1 +135,136: : +137,143: StringLiteral "name" +144,147: For +148,152: Identifier name +153,155: In +156,165: Identifier get_names +165,166: ( +166,167: ) +167,168: NL +168,169: } +169,170: NewLine +170,171: NL +171,191: Identifier aggregated_key_stats +191,192: [ +192,200: Identifier idx_stat +200,201: ] +202,203: = +204,205: { +205,206: NL +210,212: ** +212,229: Identifier filter_value_NONE +229,230: , +230,231: NL +235,237: ** +237,253: Identifier filter_key_found +253,254: , +254,255: NL +259,261: ** +261,273: Identifier client_types +273,274: , +274,275: NL +279,281: ** +281,293: Identifier market_watch +293,294: , +294,295: NL +299,307: StringLiteral "symbol" +307,308: : +309,315: Identifier symbol +315,316: , +316,317: NL +321,327: StringLiteral "name" +327,328: : +329,333: Identifier name +333,334: , +334,335: NL +335,336: } +336,337: NewLine +337,338: NL +338,340: Identifier di +341,342: = +343,344: { +344,345: Integer 1 +345,346: : +347,353: StringLiteral "name" +353,354: , +355,357: ** +357,358: { +358,359: Integer 2 +359,360: : +361,368: StringLiteral "name2" +368,369: } +369,370: , +371,372: Integer 3 +372,373: : +374,381: StringLiteral "name3" +381,382: } +382,383: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@for.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@for.py.snap index 503df88f..5ad883f9 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@for.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@for.py.snap @@ -3,111 +3,111 @@ source: parser/src/lexer/mod.rs description: "for a in b:\n pass\n\nfor a in b:\n pass\n\nfor a in range(10):\n a = 1\nelse:\n b = 1\n\nfor a in range(10), range(10):\n a = 1\n\nasync for a in b:\n pass\n\nasync for a in b:\n pass\n\nfor a in [1, 2, 3]:\n pass\n" input_file: parser/test_data/inputs/for.py --- -0,3: For (None) -4,5: Identifier (Str("a")) -6,8: In (None) -9,10: Identifier (Str("b")) -10,11: : (None) -11,12: NewLine (None) -12,16: Indent (Indent(1)) -16,20: Pass (None) -20,21: NewLine (None) -21,22: NL (None) -22,22: Dedent (Indent(1)) -22,25: For (None) -26,27: Identifier (Str("a")) -28,30: In (None) -31,32: Identifier (Str("b")) -32,33: : (None) -33,34: NewLine (None) -34,38: Indent (Indent(1)) -38,42: Pass (None) -42,43: NewLine (None) -43,44: NL (None) -44,44: Dedent (Indent(1)) -44,47: For (None) -48,49: Identifier (Str("a")) -50,52: In (None) -53,58: Identifier (Str("range")) -58,59: ( (None) -59,61: Integer (Number("10")) -61,62: ) (None) -62,63: : (None) -63,64: NewLine (None) -64,68: Indent (Indent(1)) -68,69: Identifier (Str("a")) -70,71: = (None) -72,73: Integer (Number("1")) -73,74: NewLine (None) -74,74: Dedent (Indent(1)) -74,78: Else (None) -78,79: : (None) -79,80: NewLine (None) -80,84: Indent (Indent(1)) -84,85: Identifier (Str("b")) -86,87: = (None) -88,89: Integer (Number("1")) -89,90: NewLine (None) -90,91: NL (None) -91,91: Dedent (Indent(1)) -91,94: For (None) -95,96: Identifier (Str("a")) -97,99: In (None) -100,105: Identifier (Str("range")) -105,106: ( (None) -106,108: Integer (Number("10")) -108,109: ) (None) -109,110: , (None) -111,116: Identifier (Str("range")) -116,117: ( (None) -117,119: Integer (Number("10")) -119,120: ) (None) -120,121: : (None) -121,122: NewLine (None) -122,126: Indent (Indent(1)) -126,127: Identifier (Str("a")) -128,129: = (None) -130,131: Integer (Number("1")) -131,132: NewLine (None) -132,133: NL (None) -133,133: Dedent (Indent(1)) -133,138: Async (None) -139,142: For (None) -143,144: Identifier (Str("a")) -145,147: In (None) -148,149: Identifier (Str("b")) -149,150: : (None) -150,151: NewLine (None) -151,155: Indent (Indent(1)) -155,159: Pass (None) -159,160: NewLine (None) -160,161: NL (None) -161,161: Dedent (Indent(1)) -161,166: Async (None) -167,170: For (None) -171,172: Identifier (Str("a")) -173,175: In (None) -176,177: Identifier (Str("b")) -177,178: : (None) -178,179: NewLine (None) -179,183: Indent (Indent(1)) -183,187: Pass (None) -187,188: NewLine (None) -188,189: NL (None) -189,189: Dedent (Indent(1)) -189,192: For (None) -193,194: Identifier (Str("a")) -195,197: In (None) -198,199: [ (None) -199,200: Integer (Number("1")) -200,201: , (None) -202,203: Integer (Number("2")) -203,204: , (None) -205,206: Integer (Number("3")) -206,207: ] (None) -207,208: : (None) -208,209: NewLine (None) -209,213: Indent (Indent(1)) -213,217: Pass (None) -217,218: NewLine (None) -218,218: Dedent (Indent(1)) +0,3: For +4,5: Identifier a +6,8: In +9,10: Identifier b +10,11: : +11,12: NewLine +12,16: Indent +16,20: Pass +20,21: NewLine +21,22: NL +22,22: Dedent +22,25: For +26,27: Identifier a +28,30: In +31,32: Identifier b +32,33: : +33,34: NewLine +34,38: Indent +38,42: Pass +42,43: NewLine +43,44: NL +44,44: Dedent +44,47: For +48,49: Identifier a +50,52: In +53,58: Identifier range +58,59: ( +59,61: Integer 10 +61,62: ) +62,63: : +63,64: NewLine +64,68: Indent +68,69: Identifier a +70,71: = +72,73: Integer 1 +73,74: NewLine +74,74: Dedent +74,78: Else +78,79: : +79,80: NewLine +80,84: Indent +84,85: Identifier b +86,87: = +88,89: Integer 1 +89,90: NewLine +90,91: NL +91,91: Dedent +91,94: For +95,96: Identifier a +97,99: In +100,105: Identifier range +105,106: ( +106,108: Integer 10 +108,109: ) +109,110: , +111,116: Identifier range +116,117: ( +117,119: Integer 10 +119,120: ) +120,121: : +121,122: NewLine +122,126: Indent +126,127: Identifier a +128,129: = +130,131: Integer 1 +131,132: NewLine +132,133: NL +133,133: Dedent +133,138: Async +139,142: For +143,144: Identifier a +145,147: In +148,149: Identifier b +149,150: : +150,151: NewLine +151,155: Indent +155,159: Pass +159,160: NewLine +160,161: NL +161,161: Dedent +161,166: Async +167,170: For +171,172: Identifier a +173,175: In +176,177: Identifier b +177,178: : +178,179: NewLine +179,183: Indent +183,187: Pass +187,188: NewLine +188,189: NL +189,189: Dedent +189,192: For +193,194: Identifier a +195,197: In +198,199: [ +199,200: Integer 1 +200,201: , +202,203: Integer 2 +203,204: , +205,206: Integer 3 +206,207: ] +207,208: : +208,209: NewLine +209,213: Indent +213,217: Pass +217,218: NewLine +218,218: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@from_import.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@from_import.py.snap index 67035b12..5d942922 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@from_import.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@from_import.py.snap @@ -3,98 +3,98 @@ source: parser/src/lexer/mod.rs description: "import a\nimport a as b\nimport a.b\nimport a.b as c\nimport a.b.c\nfrom a import b\nfrom a import b as c\nfrom a.b import c\nfrom a.b import c as d\nfrom ...a import b\nfrom ....a import b\nfrom .....a import b\nfrom ......a import b\nfrom .......a import b\n# TODO(parser): enable after error handling\n# from ...\n" input_file: parser/test_data/inputs/from_import.py --- -0,6: Import (None) -7,8: Identifier (Str("a")) -8,9: NewLine (None) -9,15: Import (None) -16,17: Identifier (Str("a")) -18,20: As (None) -21,22: Identifier (Str("b")) -22,23: NewLine (None) -23,29: Import (None) -30,31: Identifier (Str("a")) -31,32: . (None) -32,33: Identifier (Str("b")) -33,34: NewLine (None) -34,40: Import (None) -41,42: Identifier (Str("a")) -42,43: . (None) -43,44: Identifier (Str("b")) -45,47: As (None) -48,49: Identifier (Str("c")) -49,50: NewLine (None) -50,56: Import (None) -57,58: Identifier (Str("a")) -58,59: . (None) -59,60: Identifier (Str("b")) -60,61: . (None) -61,62: Identifier (Str("c")) -62,63: NewLine (None) -63,67: From (None) -68,69: Identifier (Str("a")) -70,76: Import (None) -77,78: Identifier (Str("b")) -78,79: NewLine (None) -79,83: From (None) -84,85: Identifier (Str("a")) -86,92: Import (None) -93,94: Identifier (Str("b")) -95,97: As (None) -98,99: Identifier (Str("c")) -99,100: NewLine (None) -100,104: From (None) -105,106: Identifier (Str("a")) -106,107: . (None) -107,108: Identifier (Str("b")) -109,115: Import (None) -116,117: Identifier (Str("c")) -117,118: NewLine (None) -118,122: From (None) -123,124: Identifier (Str("a")) -124,125: . (None) -125,126: Identifier (Str("b")) -127,133: Import (None) -134,135: Identifier (Str("c")) -136,138: As (None) -139,140: Identifier (Str("d")) -140,141: NewLine (None) -141,145: From (None) -146,149: Ellipsis (None) -149,150: Identifier (Str("a")) -151,157: Import (None) -158,159: Identifier (Str("b")) -159,160: NewLine (None) -160,164: From (None) -165,168: Ellipsis (None) -168,169: . (None) -169,170: Identifier (Str("a")) -171,177: Import (None) -178,179: Identifier (Str("b")) -179,180: NewLine (None) -180,184: From (None) -185,188: Ellipsis (None) -188,189: . (None) -189,190: . (None) -190,191: Identifier (Str("a")) -192,198: Import (None) -199,200: Identifier (Str("b")) -200,201: NewLine (None) -201,205: From (None) -206,209: Ellipsis (None) -209,212: Ellipsis (None) -212,213: Identifier (Str("a")) -214,220: Import (None) -221,222: Identifier (Str("b")) -222,223: NewLine (None) -223,227: From (None) -228,231: Ellipsis (None) -231,234: Ellipsis (None) -234,235: . (None) -235,236: Identifier (Str("a")) -237,243: Import (None) -244,245: Identifier (Str("b")) -245,246: NewLine (None) -246,289: Comment (Str("# TODO(parser): enable after error handling")) -289,290: NL (None) -290,300: Comment (Str("# from ...")) -300,301: NL (None) +0,6: Import +7,8: Identifier a +8,9: NewLine +9,15: Import +16,17: Identifier a +18,20: As +21,22: Identifier b +22,23: NewLine +23,29: Import +30,31: Identifier a +31,32: . +32,33: Identifier b +33,34: NewLine +34,40: Import +41,42: Identifier a +42,43: . +43,44: Identifier b +45,47: As +48,49: Identifier c +49,50: NewLine +50,56: Import +57,58: Identifier a +58,59: . +59,60: Identifier b +60,61: . +61,62: Identifier c +62,63: NewLine +63,67: From +68,69: Identifier a +70,76: Import +77,78: Identifier b +78,79: NewLine +79,83: From +84,85: Identifier a +86,92: Import +93,94: Identifier b +95,97: As +98,99: Identifier c +99,100: NewLine +100,104: From +105,106: Identifier a +106,107: . +107,108: Identifier b +109,115: Import +116,117: Identifier c +117,118: NewLine +118,122: From +123,124: Identifier a +124,125: . +125,126: Identifier b +127,133: Import +134,135: Identifier c +136,138: As +139,140: Identifier d +140,141: NewLine +141,145: From +146,149: Ellipsis +149,150: Identifier a +151,157: Import +158,159: Identifier b +159,160: NewLine +160,164: From +165,168: Ellipsis +168,169: . +169,170: Identifier a +171,177: Import +178,179: Identifier b +179,180: NewLine +180,184: From +185,188: Ellipsis +188,189: . +189,190: . +190,191: Identifier a +192,198: Import +199,200: Identifier b +200,201: NewLine +201,205: From +206,209: Ellipsis +209,212: Ellipsis +212,213: Identifier a +214,220: Import +221,222: Identifier b +222,223: NewLine +223,227: From +228,231: Ellipsis +231,234: Ellipsis +234,235: . +235,236: Identifier a +237,243: Import +244,245: Identifier b +245,246: NewLine +246,289: Comment +289,290: NL +290,300: Comment +300,301: NL diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py.snap index 70d6f00e..90051870 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py.snap @@ -3,442 +3,442 @@ source: parser/src/lexer/mod.rs description: "def a(): pass\n\ndef a():\n pass\n\ndef a(a, b, c): pass\n\ndef a(a, *b, **c): pass\n\ndef a(a,\nb,\nc): pass\n\n@decor\ndef a(): pass\n\n@decor\ndef f(a: 'annotation', b=1, c=2, *d, e, f=3, **g): pass\n\ndef func() -> None: pass\n\nasync def a(): pass\n\nasync def a():\n pass\n\nasync def a(a, b, c): pass\n\ndef a[T](): pass\n\ndef a[T, U](): pass\n\ndef a[T: U](): pass\n\ndef a[T: U, V: W](): pass\n\ndef a[*T](): pass\n\ndef a[T, *U](): pass\n\ndef a[T: U, *V](): pass\n\ndef a[T: U, V: W, *X](): pass\n\ndef a[**T](): pass\n\ndef a[T, **U](): pass\n\ndef a[T: U, **V](): pass\n\ndef a[T: U, V: W, **X](): pass\n\ndef a[T, *U, **V](): pass\n\ndef a[T: U, *V, **W](): pass\n\ndef dataclass_transform(\n *,\n eq_default: bool = True,\n order_default: bool = False,\n kw_only_default: bool = False,\n frozen_default: bool = False, # on 3.11, runtime accepts it as part of kwargs\n # this next annotation cannot be parsed need fix\n field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (),\n **kwargs: Any,\n) -> IdentityFunction: ...\n" input_file: parser/test_data/inputs/function_def.py --- -0,3: Def (None) -4,5: Identifier (Str("a")) -5,6: ( (None) -6,7: ) (None) -7,8: : (None) -9,13: Pass (None) -13,14: NewLine (None) -14,15: NL (None) -15,18: Def (None) -19,20: Identifier (Str("a")) -20,21: ( (None) -21,22: ) (None) -22,23: : (None) -23,24: NewLine (None) -24,28: Indent (Indent(1)) -28,32: Pass (None) -32,33: NewLine (None) -33,34: NL (None) -34,34: Dedent (Indent(1)) -34,37: Def (None) -38,39: Identifier (Str("a")) -39,40: ( (None) -40,41: Identifier (Str("a")) -41,42: , (None) -43,44: Identifier (Str("b")) -44,45: , (None) -46,47: Identifier (Str("c")) -47,48: ) (None) -48,49: : (None) -50,54: Pass (None) -54,55: NewLine (None) -55,56: NL (None) -56,59: Def (None) -60,61: Identifier (Str("a")) -61,62: ( (None) -62,63: Identifier (Str("a")) -63,64: , (None) -65,66: * (None) -66,67: Identifier (Str("b")) -67,68: , (None) -69,71: ** (None) -71,72: Identifier (Str("c")) -72,73: ) (None) -73,74: : (None) -75,79: Pass (None) -79,80: NewLine (None) -80,81: NL (None) -81,84: Def (None) -85,86: Identifier (Str("a")) -86,87: ( (None) -87,88: Identifier (Str("a")) -88,89: , (None) -89,90: NL (None) -90,91: Identifier (Str("b")) -91,92: , (None) -92,93: NL (None) -93,94: Identifier (Str("c")) -94,95: ) (None) -95,96: : (None) -97,101: Pass (None) -101,102: NewLine (None) -102,103: NL (None) -103,104: @ (None) -104,109: Identifier (Str("decor")) -109,110: NewLine (None) -110,113: Def (None) -114,115: Identifier (Str("a")) -115,116: ( (None) -116,117: ) (None) -117,118: : (None) -119,123: Pass (None) -123,124: NewLine (None) -124,125: NL (None) -125,126: @ (None) -126,131: Identifier (Str("decor")) -131,132: NewLine (None) -132,135: Def (None) -136,137: Identifier (Str("f")) -137,138: ( (None) -138,139: Identifier (Str("a")) -139,140: : (None) -141,153: StringLiteral (Str("'annotation'")) -153,154: , (None) -155,156: Identifier (Str("b")) -156,157: = (None) -157,158: Integer (Number("1")) -158,159: , (None) -160,161: Identifier (Str("c")) -161,162: = (None) -162,163: Integer (Number("2")) -163,164: , (None) -165,166: * (None) -166,167: Identifier (Str("d")) -167,168: , (None) -169,170: Identifier (Str("e")) -170,171: , (None) -172,173: Identifier (Str("f")) -173,174: = (None) -174,175: Integer (Number("3")) -175,176: , (None) -177,179: ** (None) -179,180: Identifier (Str("g")) -180,181: ) (None) -181,182: : (None) -183,187: Pass (None) -187,188: NewLine (None) -188,189: NL (None) -189,192: Def (None) -193,197: Identifier (Str("func")) -197,198: ( (None) -198,199: ) (None) -200,202: -> (None) -203,207: None (None) -207,208: : (None) -209,213: Pass (None) -213,214: NewLine (None) -214,215: NL (None) -215,220: Async (None) -221,224: Def (None) -225,226: Identifier (Str("a")) -226,227: ( (None) -227,228: ) (None) -228,229: : (None) -230,234: Pass (None) -234,235: NewLine (None) -235,236: NL (None) -236,241: Async (None) -242,245: Def (None) -246,247: Identifier (Str("a")) -247,248: ( (None) -248,249: ) (None) -249,250: : (None) -250,251: NewLine (None) -251,255: Indent (Indent(1)) -255,259: Pass (None) -259,260: NewLine (None) -260,261: NL (None) -261,261: Dedent (Indent(1)) -261,266: Async (None) -267,270: Def (None) -271,272: Identifier (Str("a")) -272,273: ( (None) -273,274: Identifier (Str("a")) -274,275: , (None) -276,277: Identifier (Str("b")) -277,278: , (None) -279,280: Identifier (Str("c")) -280,281: ) (None) -281,282: : (None) -283,287: Pass (None) -287,288: NewLine (None) -288,289: NL (None) -289,292: Def (None) -293,294: Identifier (Str("a")) -294,295: [ (None) -295,296: Identifier (Str("T")) -296,297: ] (None) -297,298: ( (None) -298,299: ) (None) -299,300: : (None) -301,305: Pass (None) -305,306: NewLine (None) -306,307: NL (None) -307,310: Def (None) -311,312: Identifier (Str("a")) -312,313: [ (None) -313,314: Identifier (Str("T")) -314,315: , (None) -316,317: Identifier (Str("U")) -317,318: ] (None) -318,319: ( (None) -319,320: ) (None) -320,321: : (None) -322,326: Pass (None) -326,327: NewLine (None) -327,328: NL (None) -328,331: Def (None) -332,333: Identifier (Str("a")) -333,334: [ (None) -334,335: Identifier (Str("T")) -335,336: : (None) -337,338: Identifier (Str("U")) -338,339: ] (None) -339,340: ( (None) -340,341: ) (None) -341,342: : (None) -343,347: Pass (None) -347,348: NewLine (None) -348,349: NL (None) -349,352: Def (None) -353,354: Identifier (Str("a")) -354,355: [ (None) -355,356: Identifier (Str("T")) -356,357: : (None) -358,359: Identifier (Str("U")) -359,360: , (None) -361,362: Identifier (Str("V")) -362,363: : (None) -364,365: Identifier (Str("W")) -365,366: ] (None) -366,367: ( (None) -367,368: ) (None) -368,369: : (None) -370,374: Pass (None) -374,375: NewLine (None) -375,376: NL (None) -376,379: Def (None) -380,381: Identifier (Str("a")) -381,382: [ (None) -382,383: * (None) -383,384: Identifier (Str("T")) -384,385: ] (None) -385,386: ( (None) -386,387: ) (None) -387,388: : (None) -389,393: Pass (None) -393,394: NewLine (None) -394,395: NL (None) -395,398: Def (None) -399,400: Identifier (Str("a")) -400,401: [ (None) -401,402: Identifier (Str("T")) -402,403: , (None) -404,405: * (None) -405,406: Identifier (Str("U")) -406,407: ] (None) -407,408: ( (None) -408,409: ) (None) -409,410: : (None) -411,415: Pass (None) -415,416: NewLine (None) -416,417: NL (None) -417,420: Def (None) -421,422: Identifier (Str("a")) -422,423: [ (None) -423,424: Identifier (Str("T")) -424,425: : (None) -426,427: Identifier (Str("U")) -427,428: , (None) -429,430: * (None) -430,431: Identifier (Str("V")) -431,432: ] (None) -432,433: ( (None) -433,434: ) (None) -434,435: : (None) -436,440: Pass (None) -440,441: NewLine (None) -441,442: NL (None) -442,445: Def (None) -446,447: Identifier (Str("a")) -447,448: [ (None) -448,449: Identifier (Str("T")) -449,450: : (None) -451,452: Identifier (Str("U")) -452,453: , (None) -454,455: Identifier (Str("V")) -455,456: : (None) -457,458: Identifier (Str("W")) -458,459: , (None) -460,461: * (None) -461,462: Identifier (Str("X")) -462,463: ] (None) -463,464: ( (None) -464,465: ) (None) -465,466: : (None) -467,471: Pass (None) -471,472: NewLine (None) -472,473: NL (None) -473,476: Def (None) -477,478: Identifier (Str("a")) -478,479: [ (None) -479,481: ** (None) -481,482: Identifier (Str("T")) -482,483: ] (None) -483,484: ( (None) -484,485: ) (None) -485,486: : (None) -487,491: Pass (None) -491,492: NewLine (None) -492,493: NL (None) -493,496: Def (None) -497,498: Identifier (Str("a")) -498,499: [ (None) -499,500: Identifier (Str("T")) -500,501: , (None) -502,504: ** (None) -504,505: Identifier (Str("U")) -505,506: ] (None) -506,507: ( (None) -507,508: ) (None) -508,509: : (None) -510,514: Pass (None) -514,515: NewLine (None) -515,516: NL (None) -516,519: Def (None) -520,521: Identifier (Str("a")) -521,522: [ (None) -522,523: Identifier (Str("T")) -523,524: : (None) -525,526: Identifier (Str("U")) -526,527: , (None) -528,530: ** (None) -530,531: Identifier (Str("V")) -531,532: ] (None) -532,533: ( (None) -533,534: ) (None) -534,535: : (None) -536,540: Pass (None) -540,541: NewLine (None) -541,542: NL (None) -542,545: Def (None) -546,547: Identifier (Str("a")) -547,548: [ (None) -548,549: Identifier (Str("T")) -549,550: : (None) -551,552: Identifier (Str("U")) -552,553: , (None) -554,555: Identifier (Str("V")) -555,556: : (None) -557,558: Identifier (Str("W")) -558,559: , (None) -560,562: ** (None) -562,563: Identifier (Str("X")) -563,564: ] (None) -564,565: ( (None) -565,566: ) (None) -566,567: : (None) -568,572: Pass (None) -572,573: NewLine (None) -573,574: NL (None) -574,577: Def (None) -578,579: Identifier (Str("a")) -579,580: [ (None) -580,581: Identifier (Str("T")) -581,582: , (None) -583,584: * (None) -584,585: Identifier (Str("U")) -585,586: , (None) -587,589: ** (None) -589,590: Identifier (Str("V")) -590,591: ] (None) -591,592: ( (None) -592,593: ) (None) -593,594: : (None) -595,599: Pass (None) -599,600: NewLine (None) -600,601: NL (None) -601,604: Def (None) -605,606: Identifier (Str("a")) -606,607: [ (None) -607,608: Identifier (Str("T")) -608,609: : (None) -610,611: Identifier (Str("U")) -611,612: , (None) -613,614: * (None) -614,615: Identifier (Str("V")) -615,616: , (None) -617,619: ** (None) -619,620: Identifier (Str("W")) -620,621: ] (None) -621,622: ( (None) -622,623: ) (None) -623,624: : (None) -625,629: Pass (None) -629,630: NewLine (None) -630,631: NL (None) -631,634: Def (None) -635,654: Identifier (Str("dataclass_transform")) -654,655: ( (None) -655,656: NL (None) -660,661: * (None) -661,662: , (None) -662,663: NL (None) -667,677: Identifier (Str("eq_default")) -677,678: : (None) -679,683: Identifier (Str("bool")) -684,685: = (None) -686,690: True (None) -690,691: , (None) -691,692: NL (None) -696,709: Identifier (Str("order_default")) -709,710: : (None) -711,715: Identifier (Str("bool")) -716,717: = (None) -718,723: False (None) -723,724: , (None) -724,725: NL (None) -729,744: Identifier (Str("kw_only_default")) -744,745: : (None) -746,750: Identifier (Str("bool")) -751,752: = (None) -753,758: False (None) -758,759: , (None) -759,760: NL (None) -764,778: Identifier (Str("frozen_default")) -778,779: : (None) -780,784: Identifier (Str("bool")) -785,786: = (None) -787,792: False (None) -792,793: , (None) -795,842: Comment (Str("# on 3.11, runtime accepts it as part of kwargs")) -842,843: NL (None) -847,895: Comment (Str("# this next annotation cannot be parsed need fix")) -895,896: NL (None) -900,916: Identifier (Str("field_specifiers")) -916,917: : (None) -918,923: Identifier (Str("tuple")) -923,924: [ (None) -924,928: Identifier (Type) -928,929: [ (None) -929,932: Identifier (Str("Any")) -932,933: ] (None) -934,935: | (None) -936,944: Identifier (Str("Callable")) -944,945: [ (None) -945,948: Ellipsis (None) -948,949: , (None) -950,953: Identifier (Str("Any")) -953,954: ] (None) -954,955: , (None) -956,959: Ellipsis (None) -959,960: ] (None) -961,962: = (None) -963,964: ( (None) -964,965: ) (None) -965,966: , (None) -966,967: NL (None) -971,973: ** (None) -973,979: Identifier (Str("kwargs")) -979,980: : (None) -981,984: Identifier (Str("Any")) -984,985: , (None) -985,986: NL (None) -986,987: ) (None) -988,990: -> (None) -991,1007: Identifier (Str("IdentityFunction")) -1007,1008: : (None) -1009,1012: Ellipsis (None) -1012,1013: NewLine (None) +0,3: Def +4,5: Identifier a +5,6: ( +6,7: ) +7,8: : +9,13: Pass +13,14: NewLine +14,15: NL +15,18: Def +19,20: Identifier a +20,21: ( +21,22: ) +22,23: : +23,24: NewLine +24,28: Indent +28,32: Pass +32,33: NewLine +33,34: NL +34,34: Dedent +34,37: Def +38,39: Identifier a +39,40: ( +40,41: Identifier a +41,42: , +43,44: Identifier b +44,45: , +46,47: Identifier c +47,48: ) +48,49: : +50,54: Pass +54,55: NewLine +55,56: NL +56,59: Def +60,61: Identifier a +61,62: ( +62,63: Identifier a +63,64: , +65,66: * +66,67: Identifier b +67,68: , +69,71: ** +71,72: Identifier c +72,73: ) +73,74: : +75,79: Pass +79,80: NewLine +80,81: NL +81,84: Def +85,86: Identifier a +86,87: ( +87,88: Identifier a +88,89: , +89,90: NL +90,91: Identifier b +91,92: , +92,93: NL +93,94: Identifier c +94,95: ) +95,96: : +97,101: Pass +101,102: NewLine +102,103: NL +103,104: @ +104,109: Identifier decor +109,110: NewLine +110,113: Def +114,115: Identifier a +115,116: ( +116,117: ) +117,118: : +119,123: Pass +123,124: NewLine +124,125: NL +125,126: @ +126,131: Identifier decor +131,132: NewLine +132,135: Def +136,137: Identifier f +137,138: ( +138,139: Identifier a +139,140: : +141,153: StringLiteral 'annotation' +153,154: , +155,156: Identifier b +156,157: = +157,158: Integer 1 +158,159: , +160,161: Identifier c +161,162: = +162,163: Integer 2 +163,164: , +165,166: * +166,167: Identifier d +167,168: , +169,170: Identifier e +170,171: , +172,173: Identifier f +173,174: = +174,175: Integer 3 +175,176: , +177,179: ** +179,180: Identifier g +180,181: ) +181,182: : +183,187: Pass +187,188: NewLine +188,189: NL +189,192: Def +193,197: Identifier func +197,198: ( +198,199: ) +200,202: -> +203,207: None +207,208: : +209,213: Pass +213,214: NewLine +214,215: NL +215,220: Async +221,224: Def +225,226: Identifier a +226,227: ( +227,228: ) +228,229: : +230,234: Pass +234,235: NewLine +235,236: NL +236,241: Async +242,245: Def +246,247: Identifier a +247,248: ( +248,249: ) +249,250: : +250,251: NewLine +251,255: Indent +255,259: Pass +259,260: NewLine +260,261: NL +261,261: Dedent +261,266: Async +267,270: Def +271,272: Identifier a +272,273: ( +273,274: Identifier a +274,275: , +276,277: Identifier b +277,278: , +279,280: Identifier c +280,281: ) +281,282: : +283,287: Pass +287,288: NewLine +288,289: NL +289,292: Def +293,294: Identifier a +294,295: [ +295,296: Identifier T +296,297: ] +297,298: ( +298,299: ) +299,300: : +301,305: Pass +305,306: NewLine +306,307: NL +307,310: Def +311,312: Identifier a +312,313: [ +313,314: Identifier T +314,315: , +316,317: Identifier U +317,318: ] +318,319: ( +319,320: ) +320,321: : +322,326: Pass +326,327: NewLine +327,328: NL +328,331: Def +332,333: Identifier a +333,334: [ +334,335: Identifier T +335,336: : +337,338: Identifier U +338,339: ] +339,340: ( +340,341: ) +341,342: : +343,347: Pass +347,348: NewLine +348,349: NL +349,352: Def +353,354: Identifier a +354,355: [ +355,356: Identifier T +356,357: : +358,359: Identifier U +359,360: , +361,362: Identifier V +362,363: : +364,365: Identifier W +365,366: ] +366,367: ( +367,368: ) +368,369: : +370,374: Pass +374,375: NewLine +375,376: NL +376,379: Def +380,381: Identifier a +381,382: [ +382,383: * +383,384: Identifier T +384,385: ] +385,386: ( +386,387: ) +387,388: : +389,393: Pass +393,394: NewLine +394,395: NL +395,398: Def +399,400: Identifier a +400,401: [ +401,402: Identifier T +402,403: , +404,405: * +405,406: Identifier U +406,407: ] +407,408: ( +408,409: ) +409,410: : +411,415: Pass +415,416: NewLine +416,417: NL +417,420: Def +421,422: Identifier a +422,423: [ +423,424: Identifier T +424,425: : +426,427: Identifier U +427,428: , +429,430: * +430,431: Identifier V +431,432: ] +432,433: ( +433,434: ) +434,435: : +436,440: Pass +440,441: NewLine +441,442: NL +442,445: Def +446,447: Identifier a +447,448: [ +448,449: Identifier T +449,450: : +451,452: Identifier U +452,453: , +454,455: Identifier V +455,456: : +457,458: Identifier W +458,459: , +460,461: * +461,462: Identifier X +462,463: ] +463,464: ( +464,465: ) +465,466: : +467,471: Pass +471,472: NewLine +472,473: NL +473,476: Def +477,478: Identifier a +478,479: [ +479,481: ** +481,482: Identifier T +482,483: ] +483,484: ( +484,485: ) +485,486: : +487,491: Pass +491,492: NewLine +492,493: NL +493,496: Def +497,498: Identifier a +498,499: [ +499,500: Identifier T +500,501: , +502,504: ** +504,505: Identifier U +505,506: ] +506,507: ( +507,508: ) +508,509: : +510,514: Pass +514,515: NewLine +515,516: NL +516,519: Def +520,521: Identifier a +521,522: [ +522,523: Identifier T +523,524: : +525,526: Identifier U +526,527: , +528,530: ** +530,531: Identifier V +531,532: ] +532,533: ( +533,534: ) +534,535: : +536,540: Pass +540,541: NewLine +541,542: NL +542,545: Def +546,547: Identifier a +547,548: [ +548,549: Identifier T +549,550: : +551,552: Identifier U +552,553: , +554,555: Identifier V +555,556: : +557,558: Identifier W +558,559: , +560,562: ** +562,563: Identifier X +563,564: ] +564,565: ( +565,566: ) +566,567: : +568,572: Pass +572,573: NewLine +573,574: NL +574,577: Def +578,579: Identifier a +579,580: [ +580,581: Identifier T +581,582: , +583,584: * +584,585: Identifier U +585,586: , +587,589: ** +589,590: Identifier V +590,591: ] +591,592: ( +592,593: ) +593,594: : +595,599: Pass +599,600: NewLine +600,601: NL +601,604: Def +605,606: Identifier a +606,607: [ +607,608: Identifier T +608,609: : +610,611: Identifier U +611,612: , +613,614: * +614,615: Identifier V +615,616: , +617,619: ** +619,620: Identifier W +620,621: ] +621,622: ( +622,623: ) +623,624: : +625,629: Pass +629,630: NewLine +630,631: NL +631,634: Def +635,654: Identifier dataclass_transform +654,655: ( +655,656: NL +660,661: * +661,662: , +662,663: NL +667,677: Identifier eq_default +677,678: : +679,683: Identifier bool +684,685: = +686,690: True +690,691: , +691,692: NL +696,709: Identifier order_default +709,710: : +711,715: Identifier bool +716,717: = +718,723: False +723,724: , +724,725: NL +729,744: Identifier kw_only_default +744,745: : +746,750: Identifier bool +751,752: = +753,758: False +758,759: , +759,760: NL +764,778: Identifier frozen_default +778,779: : +780,784: Identifier bool +785,786: = +787,792: False +792,793: , +795,842: Comment +842,843: NL +847,895: Comment +895,896: NL +900,916: Identifier field_specifiers +916,917: : +918,923: Identifier tuple +923,924: [ +924,928: type +928,929: [ +929,932: Identifier Any +932,933: ] +934,935: | +936,944: Identifier Callable +944,945: [ +945,948: Ellipsis +948,949: , +950,953: Identifier Any +953,954: ] +954,955: , +956,959: Ellipsis +959,960: ] +961,962: = +963,964: ( +964,965: ) +965,966: , +966,967: NL +971,973: ** +973,979: Identifier kwargs +979,980: : +981,984: Identifier Any +984,985: , +985,986: NL +986,987: ) +988,990: -> +991,1007: Identifier IdentityFunction +1007,1008: : +1009,1012: Ellipsis +1012,1013: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap index a4d47628..f30c3d18 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@functions.py.snap @@ -3,281 +3,281 @@ source: parser/src/lexer/mod.rs description: "def _handle_ticker_index(symbol):\n ticker_index = symbols_data.get_ticker_index(symbol)\n\n if ticker_index is None:\n market_symbol = get_symbol_info(symbol)\n if market_symbol is not None:\n symbols_data.append_symbol_to_file(market_symbol)\n ticker_index = market_symbol.index\n return ticker_index\n\n\ndef _extract_ticker_client_types_data(ticker_index: str) -> List:\n url = TSE_CLIENT_TYPE_DATA_URL.format(ticker_index)\n with requests_retry_session() as session:\n response = session.get(url, timeout=5)\n data = response.text.split(';')\n return data\n\ndef common_process(df: pd.DataFrame, date: str):\n if len(df) == 0:\n return pd.DataFrame(columns=list(api_to_orderbook_mapping.keys()))\n df.rename(columns=reversed_keys, inplace=True)\n df = df.loc[:, list(api_to_orderbook_mapping.keys())]\n df[\"datetime\"] = pd.to_datetime(\n date + \" \" + df[\"datetime\"].astype(str), format=\"%Y%m%d %H%M%S\"\n )\n df = df.sort_values([\"datetime\", \"depth\"], ascending=[True, True])\n df.set_index(\"datetime\", inplace=True)\n df.drop(columns=[\"refID\"], inplace=True)\n return df\n\n" input_file: parser/test_data/inputs/functions.py --- -0,3: Def (None) -4,24: Identifier (Str("_handle_ticker_index")) -24,25: ( (None) -25,31: Identifier (Str("symbol")) -31,32: ) (None) -32,33: : (None) -33,34: NewLine (None) -34,38: Indent (Indent(1)) -38,50: Identifier (Str("ticker_index")) -51,52: = (None) -53,65: Identifier (Str("symbols_data")) -65,66: . (None) -66,82: Identifier (Str("get_ticker_index")) -82,83: ( (None) -83,89: Identifier (Str("symbol")) -89,90: ) (None) -90,91: NewLine (None) -91,92: NL (None) -96,98: If (None) -99,111: Identifier (Str("ticker_index")) -112,114: Is (None) -115,119: None (None) -119,120: : (None) -120,121: NewLine (None) -121,129: Indent (Indent(1)) -129,142: Identifier (Str("market_symbol")) -143,144: = (None) -145,160: Identifier (Str("get_symbol_info")) -160,161: ( (None) -161,167: Identifier (Str("symbol")) -167,168: ) (None) -168,169: NewLine (None) -177,179: If (None) -180,193: Identifier (Str("market_symbol")) -194,196: Is (None) -197,200: Not (None) -201,205: None (None) -205,206: : (None) -206,207: NewLine (None) -207,219: Indent (Indent(1)) -219,231: Identifier (Str("symbols_data")) -231,232: . (None) -232,253: Identifier (Str("append_symbol_to_file")) -253,254: ( (None) -254,267: Identifier (Str("market_symbol")) -267,268: ) (None) -268,269: NewLine (None) -281,293: Identifier (Str("ticker_index")) -294,295: = (None) -296,309: Identifier (Str("market_symbol")) -309,310: . (None) -310,315: Identifier (Str("index")) -315,316: NewLine (None) -320,320: Dedent (Indent(1)) -320,320: Dedent (None) -320,326: Return (None) -327,339: Identifier (Str("ticker_index")) -339,340: NewLine (None) -340,341: NL (None) -341,342: NL (None) -342,342: Dedent (Indent(1)) -342,345: Def (None) -346,379: Identifier (Str("_extract_ticker_client_types_data")) -379,380: ( (None) -380,392: Identifier (Str("ticker_index")) -392,393: : (None) -394,397: Identifier (Str("str")) -397,398: ) (None) -399,401: -> (None) -402,406: Identifier (Str("List")) -406,407: : (None) -407,408: NewLine (None) -408,412: Indent (Indent(1)) -412,415: Identifier (Str("url")) -416,417: = (None) -418,442: Identifier (Str("TSE_CLIENT_TYPE_DATA_URL")) -442,443: . (None) -443,449: Identifier (Str("format")) -449,450: ( (None) -450,462: Identifier (Str("ticker_index")) -462,463: ) (None) -463,464: NewLine (None) -468,472: With (None) -473,495: Identifier (Str("requests_retry_session")) -495,496: ( (None) -496,497: ) (None) -498,500: As (None) -501,508: Identifier (Str("session")) -508,509: : (None) -509,510: NewLine (None) -510,518: Indent (Indent(1)) -518,526: Identifier (Str("response")) -527,528: = (None) -529,536: Identifier (Str("session")) -536,537: . (None) -537,540: Identifier (Str("get")) -540,541: ( (None) -541,544: Identifier (Str("url")) -544,545: , (None) -546,553: Identifier (Str("timeout")) -553,554: = (None) -554,555: Integer (Number("5")) -555,556: ) (None) -556,557: NewLine (None) -561,561: Dedent (Indent(1)) -561,565: Identifier (Str("data")) -566,567: = (None) -568,576: Identifier (Str("response")) -576,577: . (None) -577,581: Identifier (Str("text")) -581,582: . (None) -582,587: Identifier (Str("split")) -587,588: ( (None) -588,591: StringLiteral (Str("';'")) -591,592: ) (None) -592,593: NewLine (None) -597,603: Return (None) -604,608: Identifier (Str("data")) -608,609: NewLine (None) -609,610: NL (None) -610,610: Dedent (Indent(1)) -610,613: Def (None) -614,628: Identifier (Str("common_process")) -628,629: ( (None) -629,631: Identifier (Str("df")) -631,632: : (None) -633,635: Identifier (Str("pd")) -635,636: . (None) -636,645: Identifier (Str("DataFrame")) -645,646: , (None) -647,651: Identifier (Str("date")) -651,652: : (None) -653,656: Identifier (Str("str")) -656,657: ) (None) -657,658: : (None) -658,659: NewLine (None) -659,663: Indent (Indent(1)) -663,665: If (None) -666,669: Identifier (Str("len")) -669,670: ( (None) -670,672: Identifier (Str("df")) -672,673: ) (None) -674,676: == (None) -677,678: Integer (Number("0")) -678,679: : (None) -679,680: NewLine (None) -680,688: Indent (Indent(1)) -688,694: Return (None) -695,697: Identifier (Str("pd")) -697,698: . (None) -698,707: Identifier (Str("DataFrame")) -707,708: ( (None) -708,715: Identifier (Str("columns")) -715,716: = (None) -716,720: Identifier (Str("list")) -720,721: ( (None) -721,745: Identifier (Str("api_to_orderbook_mapping")) -745,746: . (None) -746,750: Identifier (Str("keys")) -750,751: ( (None) -751,752: ) (None) -752,753: ) (None) -753,754: ) (None) -754,755: NewLine (None) -759,759: Dedent (Indent(1)) -759,761: Identifier (Str("df")) -761,762: . (None) -762,768: Identifier (Str("rename")) -768,769: ( (None) -769,776: Identifier (Str("columns")) -776,777: = (None) -777,790: Identifier (Str("reversed_keys")) -790,791: , (None) -792,799: Identifier (Str("inplace")) -799,800: = (None) -800,804: True (None) -804,805: ) (None) -805,806: NewLine (None) -810,812: Identifier (Str("df")) -813,814: = (None) -815,817: Identifier (Str("df")) -817,818: . (None) -818,821: Identifier (Str("loc")) -821,822: [ (None) -822,823: : (None) -823,824: , (None) -825,829: Identifier (Str("list")) -829,830: ( (None) -830,854: Identifier (Str("api_to_orderbook_mapping")) -854,855: . (None) -855,859: Identifier (Str("keys")) -859,860: ( (None) -860,861: ) (None) -861,862: ) (None) -862,863: ] (None) -863,864: NewLine (None) -868,870: Identifier (Str("df")) -870,871: [ (None) -871,881: StringLiteral (Str("\"datetime\"")) -881,882: ] (None) -883,884: = (None) -885,887: Identifier (Str("pd")) -887,888: . (None) -888,899: Identifier (Str("to_datetime")) -899,900: ( (None) -900,901: NL (None) -909,913: Identifier (Str("date")) -914,915: + (None) -916,919: StringLiteral (Str("\" \"")) -920,921: + (None) -922,924: Identifier (Str("df")) -924,925: [ (None) -925,935: StringLiteral (Str("\"datetime\"")) -935,936: ] (None) -936,937: . (None) -937,943: Identifier (Str("astype")) -943,944: ( (None) -944,947: Identifier (Str("str")) -947,948: ) (None) -948,949: , (None) -950,956: Identifier (Str("format")) -956,957: = (None) -957,972: StringLiteral (Str("\"%Y%m%d %H%M%S\"")) -972,973: NL (None) -977,978: ) (None) -978,979: NewLine (None) -983,985: Identifier (Str("df")) -986,987: = (None) -988,990: Identifier (Str("df")) -990,991: . (None) -991,1002: Identifier (Str("sort_values")) -1002,1003: ( (None) -1003,1004: [ (None) -1004,1014: StringLiteral (Str("\"datetime\"")) -1014,1015: , (None) -1016,1023: StringLiteral (Str("\"depth\"")) -1023,1024: ] (None) -1024,1025: , (None) -1026,1035: Identifier (Str("ascending")) -1035,1036: = (None) -1036,1037: [ (None) -1037,1041: True (None) -1041,1042: , (None) -1043,1047: True (None) -1047,1048: ] (None) -1048,1049: ) (None) -1049,1050: NewLine (None) -1054,1056: Identifier (Str("df")) -1056,1057: . (None) -1057,1066: Identifier (Str("set_index")) -1066,1067: ( (None) -1067,1077: StringLiteral (Str("\"datetime\"")) -1077,1078: , (None) -1079,1086: Identifier (Str("inplace")) -1086,1087: = (None) -1087,1091: True (None) -1091,1092: ) (None) -1092,1093: NewLine (None) -1097,1099: Identifier (Str("df")) -1099,1100: . (None) -1100,1104: Identifier (Str("drop")) -1104,1105: ( (None) -1105,1112: Identifier (Str("columns")) -1112,1113: = (None) -1113,1114: [ (None) -1114,1121: StringLiteral (Str("\"refID\"")) -1121,1122: ] (None) -1122,1123: , (None) -1124,1131: Identifier (Str("inplace")) -1131,1132: = (None) -1132,1136: True (None) -1136,1137: ) (None) -1137,1138: NewLine (None) -1142,1148: Return (None) -1149,1151: Identifier (Str("df")) -1151,1152: NewLine (None) -1152,1152: Dedent (Indent(1)) -1152,1153: NL (None) +0,3: Def +4,24: Identifier _handle_ticker_index +24,25: ( +25,31: Identifier symbol +31,32: ) +32,33: : +33,34: NewLine +34,38: Indent +38,50: Identifier ticker_index +51,52: = +53,65: Identifier symbols_data +65,66: . +66,82: Identifier get_ticker_index +82,83: ( +83,89: Identifier symbol +89,90: ) +90,91: NewLine +91,92: NL +96,98: If +99,111: Identifier ticker_index +112,114: Is +115,119: None +119,120: : +120,121: NewLine +121,129: Indent +129,142: Identifier market_symbol +143,144: = +145,160: Identifier get_symbol_info +160,161: ( +161,167: Identifier symbol +167,168: ) +168,169: NewLine +177,179: If +180,193: Identifier market_symbol +194,196: Is +197,200: Not +201,205: None +205,206: : +206,207: NewLine +207,219: Indent +219,231: Identifier symbols_data +231,232: . +232,253: Identifier append_symbol_to_file +253,254: ( +254,267: Identifier market_symbol +267,268: ) +268,269: NewLine +281,293: Identifier ticker_index +294,295: = +296,309: Identifier market_symbol +309,310: . +310,315: Identifier index +315,316: NewLine +320,320: Dedent +320,320: Dedent +320,326: Return +327,339: Identifier ticker_index +339,340: NewLine +340,341: NL +341,342: NL +342,342: Dedent +342,345: Def +346,379: Identifier _extract_ticker_client_types_data +379,380: ( +380,392: Identifier ticker_index +392,393: : +394,397: Identifier str +397,398: ) +399,401: -> +402,406: Identifier List +406,407: : +407,408: NewLine +408,412: Indent +412,415: Identifier url +416,417: = +418,442: Identifier TSE_CLIENT_TYPE_DATA_URL +442,443: . +443,449: Identifier format +449,450: ( +450,462: Identifier ticker_index +462,463: ) +463,464: NewLine +468,472: With +473,495: Identifier requests_retry_session +495,496: ( +496,497: ) +498,500: As +501,508: Identifier session +508,509: : +509,510: NewLine +510,518: Indent +518,526: Identifier response +527,528: = +529,536: Identifier session +536,537: . +537,540: Identifier get +540,541: ( +541,544: Identifier url +544,545: , +546,553: Identifier timeout +553,554: = +554,555: Integer 5 +555,556: ) +556,557: NewLine +561,561: Dedent +561,565: Identifier data +566,567: = +568,576: Identifier response +576,577: . +577,581: Identifier text +581,582: . +582,587: Identifier split +587,588: ( +588,591: StringLiteral ';' +591,592: ) +592,593: NewLine +597,603: Return +604,608: Identifier data +608,609: NewLine +609,610: NL +610,610: Dedent +610,613: Def +614,628: Identifier common_process +628,629: ( +629,631: Identifier df +631,632: : +633,635: Identifier pd +635,636: . +636,645: Identifier DataFrame +645,646: , +647,651: Identifier date +651,652: : +653,656: Identifier str +656,657: ) +657,658: : +658,659: NewLine +659,663: Indent +663,665: If +666,669: Identifier len +669,670: ( +670,672: Identifier df +672,673: ) +674,676: == +677,678: Integer 0 +678,679: : +679,680: NewLine +680,688: Indent +688,694: Return +695,697: Identifier pd +697,698: . +698,707: Identifier DataFrame +707,708: ( +708,715: Identifier columns +715,716: = +716,720: Identifier list +720,721: ( +721,745: Identifier api_to_orderbook_mapping +745,746: . +746,750: Identifier keys +750,751: ( +751,752: ) +752,753: ) +753,754: ) +754,755: NewLine +759,759: Dedent +759,761: Identifier df +761,762: . +762,768: Identifier rename +768,769: ( +769,776: Identifier columns +776,777: = +777,790: Identifier reversed_keys +790,791: , +792,799: Identifier inplace +799,800: = +800,804: True +804,805: ) +805,806: NewLine +810,812: Identifier df +813,814: = +815,817: Identifier df +817,818: . +818,821: Identifier loc +821,822: [ +822,823: : +823,824: , +825,829: Identifier list +829,830: ( +830,854: Identifier api_to_orderbook_mapping +854,855: . +855,859: Identifier keys +859,860: ( +860,861: ) +861,862: ) +862,863: ] +863,864: NewLine +868,870: Identifier df +870,871: [ +871,881: StringLiteral "datetime" +881,882: ] +883,884: = +885,887: Identifier pd +887,888: . +888,899: Identifier to_datetime +899,900: ( +900,901: NL +909,913: Identifier date +914,915: + +916,919: StringLiteral " " +920,921: + +922,924: Identifier df +924,925: [ +925,935: StringLiteral "datetime" +935,936: ] +936,937: . +937,943: Identifier astype +943,944: ( +944,947: Identifier str +947,948: ) +948,949: , +950,956: Identifier format +956,957: = +957,972: StringLiteral "%Y%m%d %H%M%S" +972,973: NL +977,978: ) +978,979: NewLine +983,985: Identifier df +986,987: = +988,990: Identifier df +990,991: . +991,1002: Identifier sort_values +1002,1003: ( +1003,1004: [ +1004,1014: StringLiteral "datetime" +1014,1015: , +1016,1023: StringLiteral "depth" +1023,1024: ] +1024,1025: , +1026,1035: Identifier ascending +1035,1036: = +1036,1037: [ +1037,1041: True +1041,1042: , +1043,1047: True +1047,1048: ] +1048,1049: ) +1049,1050: NewLine +1054,1056: Identifier df +1056,1057: . +1057,1066: Identifier set_index +1066,1067: ( +1067,1077: StringLiteral "datetime" +1077,1078: , +1079,1086: Identifier inplace +1086,1087: = +1087,1091: True +1091,1092: ) +1092,1093: NewLine +1097,1099: Identifier df +1099,1100: . +1100,1104: Identifier drop +1104,1105: ( +1105,1112: Identifier columns +1112,1113: = +1113,1114: [ +1114,1121: StringLiteral "refID" +1121,1122: ] +1122,1123: , +1124,1131: Identifier inplace +1131,1132: = +1132,1136: True +1136,1137: ) +1137,1138: NewLine +1142,1148: Return +1149,1151: Identifier df +1151,1152: NewLine +1152,1152: Dedent +1152,1153: NL diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@generator_expressions.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@generator_expressions.py.snap index 1e41bd6c..ab9f96d9 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@generator_expressions.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@generator_expressions.py.snap @@ -3,130 +3,130 @@ source: parser/src/lexer/mod.rs description: "(a for a in b)\n\n(a for a in b if c)\n\n(a for a in b if c if d)\n\n(a for a in b for c in d)\n\n(ord(c) for line in file for c in line)\n\n(x*y for x in range(10) for y in range(10) if x*y > 10)\n\nhas_explicit_annotation = isinstance(fdef.type, CallableType) and any(\n not is_unannotated_any(t) for t in fdef.type.arg_types + [fdef.type.ret_type]\n)\n" input_file: parser/test_data/inputs/generator_expressions.py --- -0,1: ( (None) -1,2: Identifier (Str("a")) -3,6: For (None) -7,8: Identifier (Str("a")) -9,11: In (None) -12,13: Identifier (Str("b")) -13,14: ) (None) -14,15: NewLine (None) -15,16: NL (None) -16,17: ( (None) -17,18: Identifier (Str("a")) -19,22: For (None) -23,24: Identifier (Str("a")) -25,27: In (None) -28,29: Identifier (Str("b")) -30,32: If (None) -33,34: Identifier (Str("c")) -34,35: ) (None) -35,36: NewLine (None) -36,37: NL (None) -37,38: ( (None) -38,39: Identifier (Str("a")) -40,43: For (None) -44,45: Identifier (Str("a")) -46,48: In (None) -49,50: Identifier (Str("b")) -51,53: If (None) -54,55: Identifier (Str("c")) -56,58: If (None) -59,60: Identifier (Str("d")) -60,61: ) (None) -61,62: NewLine (None) -62,63: NL (None) -63,64: ( (None) -64,65: Identifier (Str("a")) -66,69: For (None) -70,71: Identifier (Str("a")) -72,74: In (None) -75,76: Identifier (Str("b")) -77,80: For (None) -81,82: Identifier (Str("c")) -83,85: In (None) -86,87: Identifier (Str("d")) -87,88: ) (None) -88,89: NewLine (None) -89,90: NL (None) -90,91: ( (None) -91,94: Identifier (Str("ord")) -94,95: ( (None) -95,96: Identifier (Str("c")) -96,97: ) (None) -98,101: For (None) -102,106: Identifier (Str("line")) -107,109: In (None) -110,114: Identifier (Str("file")) -115,118: For (None) -119,120: Identifier (Str("c")) -121,123: In (None) -124,128: Identifier (Str("line")) -128,129: ) (None) -129,130: NewLine (None) -130,131: NL (None) -131,132: ( (None) -132,133: Identifier (Str("x")) -133,134: * (None) -134,135: Identifier (Str("y")) -136,139: For (None) -140,141: Identifier (Str("x")) -142,144: In (None) -145,150: Identifier (Str("range")) -150,151: ( (None) -151,153: Integer (Number("10")) -153,154: ) (None) -155,158: For (None) -159,160: Identifier (Str("y")) -161,163: In (None) -164,169: Identifier (Str("range")) -169,170: ( (None) -170,172: Integer (Number("10")) -172,173: ) (None) -174,176: If (None) -177,178: Identifier (Str("x")) -178,179: * (None) -179,180: Identifier (Str("y")) -181,182: > (None) -183,185: Integer (Number("10")) -185,186: ) (None) -186,187: NewLine (None) -187,188: NL (None) -188,211: Identifier (Str("has_explicit_annotation")) -212,213: = (None) -214,224: Identifier (Str("isinstance")) -224,225: ( (None) -225,229: Identifier (Str("fdef")) -229,230: . (None) -230,234: Identifier (Type) -234,235: , (None) -236,248: Identifier (Str("CallableType")) -248,249: ) (None) -250,253: And (None) -254,257: Identifier (Str("any")) -257,258: ( (None) -258,259: NL (None) -263,266: Not (None) -267,285: Identifier (Str("is_unannotated_any")) -285,286: ( (None) -286,287: Identifier (Str("t")) -287,288: ) (None) -289,292: For (None) -293,294: Identifier (Str("t")) -295,297: In (None) -298,302: Identifier (Str("fdef")) -302,303: . (None) -303,307: Identifier (Type) -307,308: . (None) -308,317: Identifier (Str("arg_types")) -318,319: + (None) -320,321: [ (None) -321,325: Identifier (Str("fdef")) -325,326: . (None) -326,330: Identifier (Type) -330,331: . (None) -331,339: Identifier (Str("ret_type")) -339,340: ] (None) -340,341: NL (None) -341,342: ) (None) -342,343: NewLine (None) +0,1: ( +1,2: Identifier a +3,6: For +7,8: Identifier a +9,11: In +12,13: Identifier b +13,14: ) +14,15: NewLine +15,16: NL +16,17: ( +17,18: Identifier a +19,22: For +23,24: Identifier a +25,27: In +28,29: Identifier b +30,32: If +33,34: Identifier c +34,35: ) +35,36: NewLine +36,37: NL +37,38: ( +38,39: Identifier a +40,43: For +44,45: Identifier a +46,48: In +49,50: Identifier b +51,53: If +54,55: Identifier c +56,58: If +59,60: Identifier d +60,61: ) +61,62: NewLine +62,63: NL +63,64: ( +64,65: Identifier a +66,69: For +70,71: Identifier a +72,74: In +75,76: Identifier b +77,80: For +81,82: Identifier c +83,85: In +86,87: Identifier d +87,88: ) +88,89: NewLine +89,90: NL +90,91: ( +91,94: Identifier ord +94,95: ( +95,96: Identifier c +96,97: ) +98,101: For +102,106: Identifier line +107,109: In +110,114: Identifier file +115,118: For +119,120: Identifier c +121,123: In +124,128: Identifier line +128,129: ) +129,130: NewLine +130,131: NL +131,132: ( +132,133: Identifier x +133,134: * +134,135: Identifier y +136,139: For +140,141: Identifier x +142,144: In +145,150: Identifier range +150,151: ( +151,153: Integer 10 +153,154: ) +155,158: For +159,160: Identifier y +161,163: In +164,169: Identifier range +169,170: ( +170,172: Integer 10 +172,173: ) +174,176: If +177,178: Identifier x +178,179: * +179,180: Identifier y +181,182: > +183,185: Integer 10 +185,186: ) +186,187: NewLine +187,188: NL +188,211: Identifier has_explicit_annotation +212,213: = +214,224: Identifier isinstance +224,225: ( +225,229: Identifier fdef +229,230: . +230,234: type +234,235: , +236,248: Identifier CallableType +248,249: ) +250,253: And +254,257: Identifier any +257,258: ( +258,259: NL +263,266: Not +267,285: Identifier is_unannotated_any +285,286: ( +286,287: Identifier t +287,288: ) +289,292: For +293,294: Identifier t +295,297: In +298,302: Identifier fdef +302,303: . +303,307: type +307,308: . +308,317: Identifier arg_types +318,319: + +320,321: [ +321,325: Identifier fdef +325,326: . +326,330: type +330,331: . +331,339: Identifier ret_type +339,340: ] +340,341: NL +341,342: ) +342,343: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap index e1cfdbcd..fd73a367 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@if.py.snap @@ -3,160 +3,160 @@ source: parser/src/lexer/mod.rs description: "if a:\n pass\n\nif a:\n\n pass\n\nif a: pass\n\nif a:\n pass;pass\n\nif a is b:\n pass\n\nif a is b:\n pass\nelif a is c:\n pass\n\nif a is b:\n pass\nelif a is c:\n pass\nelse:\n pass\n\n\nif a:\n a = 1\nif a:\n b = 1\n\nif self.adjust:\n if True:\n print(\"adjust is True and adjusted_daily_records_csv_path exists\")\n else:\n print(\"adjust is True\")\nelse:\n print(\"adjust is False\")\n" input_file: parser/test_data/inputs/if.py --- -0,2: If (None) -3,4: Identifier (Str("a")) -4,5: : (None) -5,6: NewLine (None) -6,10: Indent (Indent(1)) -10,14: Pass (None) -14,15: NewLine (None) -15,16: NL (None) -16,16: Dedent (Indent(1)) -16,18: If (None) -19,20: Identifier (Str("a")) -20,21: : (None) -21,22: NewLine (None) -22,23: NL (None) -23,27: Indent (Indent(1)) -27,31: Pass (None) -31,32: NewLine (None) -32,33: NL (None) -33,33: Dedent (Indent(1)) -33,35: If (None) -36,37: Identifier (Str("a")) -37,38: : (None) -39,43: Pass (None) -43,44: NewLine (None) -44,45: NL (None) -45,47: If (None) -48,49: Identifier (Str("a")) -49,50: : (None) -50,51: NewLine (None) -51,55: Indent (Indent(1)) -55,59: Pass (None) -59,60: ; (None) -60,64: Pass (None) -64,65: NewLine (None) -65,66: NL (None) -66,66: Dedent (Indent(1)) -66,68: If (None) -69,70: Identifier (Str("a")) -71,73: Is (None) -74,75: Identifier (Str("b")) -75,76: : (None) -76,77: NewLine (None) -77,89: Indent (Indent(1)) -89,93: Pass (None) -93,94: NewLine (None) -94,95: NL (None) -95,95: Dedent (Indent(1)) -95,97: If (None) -98,99: Identifier (Str("a")) -100,102: Is (None) -103,104: Identifier (Str("b")) -104,105: : (None) -105,106: NewLine (None) -106,122: Indent (Indent(1)) -122,126: Pass (None) -126,127: NewLine (None) -127,127: Dedent (Indent(1)) -127,131: Elif (None) -132,133: Identifier (Str("a")) -134,136: Is (None) -137,138: Identifier (Str("c")) -138,139: : (None) -139,140: NewLine (None) -140,156: Indent (Indent(1)) -156,160: Pass (None) -160,161: NewLine (None) -161,162: NL (None) -162,162: Dedent (Indent(1)) -162,164: If (None) -165,166: Identifier (Str("a")) -167,169: Is (None) -170,171: Identifier (Str("b")) -171,172: : (None) -172,173: NewLine (None) -173,189: Indent (Indent(1)) -189,193: Pass (None) -193,194: NewLine (None) -194,194: Dedent (Indent(1)) -194,198: Elif (None) -199,200: Identifier (Str("a")) -201,203: Is (None) -204,205: Identifier (Str("c")) -205,206: : (None) -206,207: NewLine (None) -207,223: Indent (Indent(1)) -223,227: Pass (None) -227,228: NewLine (None) -228,228: Dedent (Indent(1)) -228,232: Else (None) -232,233: : (None) -233,234: NewLine (None) -234,250: Indent (Indent(1)) -250,254: Pass (None) -254,255: NewLine (None) -255,256: NL (None) -256,257: NL (None) -257,257: Dedent (Indent(1)) -257,259: If (None) -260,261: Identifier (Str("a")) -261,262: : (None) -262,263: NewLine (None) -263,267: Indent (Indent(1)) -267,268: Identifier (Str("a")) -269,270: = (None) -271,272: Integer (Number("1")) -272,273: NewLine (None) -273,273: Dedent (Indent(1)) -273,275: If (None) -276,277: Identifier (Str("a")) -277,278: : (None) -278,279: NewLine (None) -279,283: Indent (Indent(1)) -283,284: Identifier (Str("b")) -285,286: = (None) -287,288: Integer (Number("1")) -288,289: NewLine (None) -289,290: NL (None) -290,290: Dedent (Indent(1)) -290,292: If (None) -293,297: Identifier (Str("self")) -297,298: . (None) -298,304: Identifier (Str("adjust")) -304,305: : (None) -305,306: NewLine (None) -306,310: Indent (Indent(1)) -310,312: If (None) -313,317: True (None) -317,318: : (None) -318,319: NewLine (None) -319,327: Indent (Indent(1)) -327,332: Identifier (Str("print")) -332,333: ( (None) -333,392: StringLiteral (Str("\"adjust is True and adjusted_daily_records_csv_path exists\"")) -392,393: ) (None) -393,394: NewLine (None) -398,398: Dedent (Indent(1)) -398,402: Else (None) -402,403: : (None) -403,404: NewLine (None) -404,412: Indent (Indent(1)) -412,417: Identifier (Str("print")) -417,418: ( (None) -418,434: StringLiteral (Str("\"adjust is True\"")) -434,435: ) (None) -435,436: NewLine (None) -436,436: Dedent (Indent(1)) -436,436: Dedent (None) -436,440: Else (None) -440,441: : (None) -441,442: NewLine (None) -442,446: Indent (Indent(1)) -446,451: Identifier (Str("print")) -451,452: ( (None) -452,469: StringLiteral (Str("\"adjust is False\"")) -469,470: ) (None) -470,471: NewLine (None) -471,471: Dedent (Indent(1)) +0,2: If +3,4: Identifier a +4,5: : +5,6: NewLine +6,10: Indent +10,14: Pass +14,15: NewLine +15,16: NL +16,16: Dedent +16,18: If +19,20: Identifier a +20,21: : +21,22: NewLine +22,23: NL +23,27: Indent +27,31: Pass +31,32: NewLine +32,33: NL +33,33: Dedent +33,35: If +36,37: Identifier a +37,38: : +39,43: Pass +43,44: NewLine +44,45: NL +45,47: If +48,49: Identifier a +49,50: : +50,51: NewLine +51,55: Indent +55,59: Pass +59,60: ; +60,64: Pass +64,65: NewLine +65,66: NL +66,66: Dedent +66,68: If +69,70: Identifier a +71,73: Is +74,75: Identifier b +75,76: : +76,77: NewLine +77,89: Indent +89,93: Pass +93,94: NewLine +94,95: NL +95,95: Dedent +95,97: If +98,99: Identifier a +100,102: Is +103,104: Identifier b +104,105: : +105,106: NewLine +106,122: Indent +122,126: Pass +126,127: NewLine +127,127: Dedent +127,131: Elif +132,133: Identifier a +134,136: Is +137,138: Identifier c +138,139: : +139,140: NewLine +140,156: Indent +156,160: Pass +160,161: NewLine +161,162: NL +162,162: Dedent +162,164: If +165,166: Identifier a +167,169: Is +170,171: Identifier b +171,172: : +172,173: NewLine +173,189: Indent +189,193: Pass +193,194: NewLine +194,194: Dedent +194,198: Elif +199,200: Identifier a +201,203: Is +204,205: Identifier c +205,206: : +206,207: NewLine +207,223: Indent +223,227: Pass +227,228: NewLine +228,228: Dedent +228,232: Else +232,233: : +233,234: NewLine +234,250: Indent +250,254: Pass +254,255: NewLine +255,256: NL +256,257: NL +257,257: Dedent +257,259: If +260,261: Identifier a +261,262: : +262,263: NewLine +263,267: Indent +267,268: Identifier a +269,270: = +271,272: Integer 1 +272,273: NewLine +273,273: Dedent +273,275: If +276,277: Identifier a +277,278: : +278,279: NewLine +279,283: Indent +283,284: Identifier b +285,286: = +287,288: Integer 1 +288,289: NewLine +289,290: NL +290,290: Dedent +290,292: If +293,297: Identifier self +297,298: . +298,304: Identifier adjust +304,305: : +305,306: NewLine +306,310: Indent +310,312: If +313,317: True +317,318: : +318,319: NewLine +319,327: Indent +327,332: Identifier print +332,333: ( +333,392: StringLiteral "adjust is True and adjusted_daily_records_csv_path exists" +392,393: ) +393,394: NewLine +398,398: Dedent +398,402: Else +402,403: : +403,404: NewLine +404,412: Indent +412,417: Identifier print +417,418: ( +418,434: StringLiteral "adjust is True" +434,435: ) +435,436: NewLine +436,436: Dedent +436,436: Dedent +436,440: Else +440,441: : +441,442: NewLine +442,446: Indent +446,451: Identifier print +451,452: ( +452,469: StringLiteral "adjust is False" +469,470: ) +470,471: NewLine +471,471: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@indentation.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@indentation.py.snap index e22012f7..abac4dfd 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@indentation.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@indentation.py.snap @@ -3,11 +3,11 @@ source: parser/src/lexer/mod.rs description: "# Unindent does not match any outer indentation level\n# if True:\n# pass\n# pass\n" input_file: parser/test_data/inputs/indentation.py --- -0,53: Comment (Str("# Unindent does not match any outer indentation level")) -53,54: NL (None) -54,64: Comment (Str("# if True:")) -64,65: NL (None) -65,79: Comment (Str("# pass")) -79,80: NL (None) -80,90: Comment (Str("# pass")) -90,91: NL (None) +0,53: Comment +53,54: NL +54,64: Comment +64,65: NL +65,79: Comment +79,80: NL +80,90: Comment +90,91: NL diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@lists.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@lists.py.snap index dd44fa4b..fcec5913 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@lists.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@lists.py.snap @@ -3,161 +3,161 @@ source: parser/src/lexer/mod.rs description: "\n[a, b, c]\n\n[a,\nb, c],\n\n[a\n, b, c],\n\n[a,\nb,\n c],\n\n[a,\n],\n\n[a, b, c,]\n\nmonth_names = ['Januari', 'Februari', 'Maart', # These are the\n 'April', 'Mei', 'June', # Dutch names\n 'July', 'Augustus', 'September', # for the months\n 'October', 'November', 'December'] # of the year,\n\n[a for a in b]\n\n[a for a in b if c]\n\n[a for a in b if c if d]\n\n[a for a in b for c in d]\n\n[a for a in b if c for d in e]\n" input_file: parser/test_data/inputs/lists.py --- -0,1: NL (None) -1,2: [ (None) -2,3: Identifier (Str("a")) -3,4: , (None) -5,6: Identifier (Str("b")) -6,7: , (None) -8,9: Identifier (Str("c")) -9,10: ] (None) -10,11: NewLine (None) -11,12: NL (None) -12,13: [ (None) -13,14: Identifier (Str("a")) -14,15: , (None) -15,16: NL (None) -16,17: Identifier (Str("b")) -17,18: , (None) -19,20: Identifier (Str("c")) -20,21: ] (None) -21,22: , (None) -22,23: NewLine (None) -23,24: NL (None) -24,25: [ (None) -25,26: Identifier (Str("a")) -26,27: NL (None) -27,28: , (None) -29,30: Identifier (Str("b")) -30,31: , (None) -32,33: Identifier (Str("c")) -33,34: ] (None) -34,35: , (None) -35,36: NewLine (None) -36,37: NL (None) -37,38: [ (None) -38,39: Identifier (Str("a")) -39,40: , (None) -40,41: NL (None) -41,42: Identifier (Str("b")) -42,43: , (None) -43,44: NL (None) -48,49: Identifier (Str("c")) -49,50: ] (None) -50,51: , (None) -51,52: NewLine (None) -52,53: NL (None) -53,54: [ (None) -54,55: Identifier (Str("a")) -55,56: , (None) -56,57: NL (None) -57,58: ] (None) -58,59: , (None) -59,60: NewLine (None) -60,61: NL (None) -61,62: [ (None) -62,63: Identifier (Str("a")) -63,64: , (None) -65,66: Identifier (Str("b")) -66,67: , (None) -68,69: Identifier (Str("c")) -69,70: , (None) -70,71: ] (None) -71,72: NewLine (None) -72,73: NL (None) -73,84: Identifier (Str("month_names")) -85,86: = (None) -87,88: [ (None) -88,97: StringLiteral (Str("'Januari'")) -97,98: , (None) -99,109: StringLiteral (Str("'Februari'")) -109,110: , (None) -111,118: StringLiteral (Str("'Maart'")) -118,119: , (None) -125,140: Comment (Str("# These are the")) -140,141: NL (None) -144,151: StringLiteral (Str("'April'")) -151,152: , (None) -155,160: StringLiteral (Str("'Mei'")) -160,161: , (None) -167,173: StringLiteral (Str("'June'")) -173,174: , (None) -181,194: Comment (Str("# Dutch names")) -194,195: NL (None) -198,204: StringLiteral (Str("'July'")) -204,205: , (None) -209,219: StringLiteral (Str("'Augustus'")) -219,220: , (None) -221,232: StringLiteral (Str("'September'")) -232,233: , (None) -235,251: Comment (Str("# for the months")) -251,252: NL (None) -255,264: StringLiteral (Str("'October'")) -264,265: , (None) -266,276: StringLiteral (Str("'November'")) -276,277: , (None) -278,288: StringLiteral (Str("'December'")) -288,289: ] (None) -292,306: Comment (Str("# of the year,")) -306,307: NewLine (None) -307,308: NL (None) -308,309: [ (None) -309,310: Identifier (Str("a")) -311,314: For (None) -315,316: Identifier (Str("a")) -317,319: In (None) -320,321: Identifier (Str("b")) -321,322: ] (None) -322,323: NewLine (None) -323,324: NL (None) -324,325: [ (None) -325,326: Identifier (Str("a")) -327,330: For (None) -331,332: Identifier (Str("a")) -333,335: In (None) -336,337: Identifier (Str("b")) -338,340: If (None) -341,342: Identifier (Str("c")) -342,343: ] (None) -343,344: NewLine (None) -344,345: NL (None) -345,346: [ (None) -346,347: Identifier (Str("a")) -348,351: For (None) -352,353: Identifier (Str("a")) -354,356: In (None) -357,358: Identifier (Str("b")) -359,361: If (None) -362,363: Identifier (Str("c")) -364,366: If (None) -367,368: Identifier (Str("d")) -368,369: ] (None) -369,370: NewLine (None) -370,371: NL (None) -371,372: [ (None) -372,373: Identifier (Str("a")) -374,377: For (None) -378,379: Identifier (Str("a")) -380,382: In (None) -383,384: Identifier (Str("b")) -385,388: For (None) -389,390: Identifier (Str("c")) -391,393: In (None) -394,395: Identifier (Str("d")) -395,396: ] (None) -396,397: NewLine (None) -397,398: NL (None) -398,399: [ (None) -399,400: Identifier (Str("a")) -401,404: For (None) -405,406: Identifier (Str("a")) -407,409: In (None) -410,411: Identifier (Str("b")) -412,414: If (None) -415,416: Identifier (Str("c")) -417,420: For (None) -421,422: Identifier (Str("d")) -423,425: In (None) -426,427: Identifier (Str("e")) -427,428: ] (None) -428,429: NewLine (None) +0,1: NL +1,2: [ +2,3: Identifier a +3,4: , +5,6: Identifier b +6,7: , +8,9: Identifier c +9,10: ] +10,11: NewLine +11,12: NL +12,13: [ +13,14: Identifier a +14,15: , +15,16: NL +16,17: Identifier b +17,18: , +19,20: Identifier c +20,21: ] +21,22: , +22,23: NewLine +23,24: NL +24,25: [ +25,26: Identifier a +26,27: NL +27,28: , +29,30: Identifier b +30,31: , +32,33: Identifier c +33,34: ] +34,35: , +35,36: NewLine +36,37: NL +37,38: [ +38,39: Identifier a +39,40: , +40,41: NL +41,42: Identifier b +42,43: , +43,44: NL +48,49: Identifier c +49,50: ] +50,51: , +51,52: NewLine +52,53: NL +53,54: [ +54,55: Identifier a +55,56: , +56,57: NL +57,58: ] +58,59: , +59,60: NewLine +60,61: NL +61,62: [ +62,63: Identifier a +63,64: , +65,66: Identifier b +66,67: , +68,69: Identifier c +69,70: , +70,71: ] +71,72: NewLine +72,73: NL +73,84: Identifier month_names +85,86: = +87,88: [ +88,97: StringLiteral 'Januari' +97,98: , +99,109: StringLiteral 'Februari' +109,110: , +111,118: StringLiteral 'Maart' +118,119: , +125,140: Comment +140,141: NL +144,151: StringLiteral 'April' +151,152: , +155,160: StringLiteral 'Mei' +160,161: , +167,173: StringLiteral 'June' +173,174: , +181,194: Comment +194,195: NL +198,204: StringLiteral 'July' +204,205: , +209,219: StringLiteral 'Augustus' +219,220: , +221,232: StringLiteral 'September' +232,233: , +235,251: Comment +251,252: NL +255,264: StringLiteral 'October' +264,265: , +266,276: StringLiteral 'November' +276,277: , +278,288: StringLiteral 'December' +288,289: ] +292,306: Comment +306,307: NewLine +307,308: NL +308,309: [ +309,310: Identifier a +311,314: For +315,316: Identifier a +317,319: In +320,321: Identifier b +321,322: ] +322,323: NewLine +323,324: NL +324,325: [ +325,326: Identifier a +327,330: For +331,332: Identifier a +333,335: In +336,337: Identifier b +338,340: If +341,342: Identifier c +342,343: ] +343,344: NewLine +344,345: NL +345,346: [ +346,347: Identifier a +348,351: For +352,353: Identifier a +354,356: In +357,358: Identifier b +359,361: If +362,363: Identifier c +364,366: If +367,368: Identifier d +368,369: ] +369,370: NewLine +370,371: NL +371,372: [ +372,373: Identifier a +374,377: For +378,379: Identifier a +380,382: In +383,384: Identifier b +385,388: For +389,390: Identifier c +391,393: In +394,395: Identifier d +395,396: ] +396,397: NewLine +397,398: NL +398,399: [ +399,400: Identifier a +401,404: For +405,406: Identifier a +407,409: In +410,411: Identifier b +412,414: If +415,416: Identifier c +417,420: For +421,422: Identifier d +423,425: In +426,427: Identifier e +427,428: ] +428,429: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@match.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@match.py.snap index f09c966b..e59b6187 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@match.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@match.py.snap @@ -3,240 +3,240 @@ source: parser/src/lexer/mod.rs description: "match a:\n case 1:\n pass\n\nmatch a:\n case 1 | 2:\n pass\n\nmatch a.b:\n case 1:\n pass\n\nmatch a:\n case None:\n pass\n case True:\n pass\n case False:\n pass\n case -1:\n pass\n case 1.0:\n pass\n case _:\n pass\n\nmatch a:\n case a.b:\n pass\n case a:\n pass\n\nmatch a:\n case (a, b):\n pass\n case {1: _ ,2: _}:\n pass\n case {**rest}:\n pass\n\nmatch x:\n case Point2D(0, 0):\n pass\n case Point3D(x=0, y=0, z=0):\n pass\n\nmatch x:\n case [a, b, c]:\n pass\n" input_file: parser/test_data/inputs/match.py --- -0,5: Identifier (Match) -6,7: Identifier (Str("a")) -7,8: : (None) -8,9: NewLine (None) -9,13: Indent (Indent(1)) -13,17: Identifier (Str("case")) -18,19: Integer (Number("1")) -19,20: : (None) -20,21: NewLine (None) -21,29: Indent (Indent(1)) -29,33: Pass (None) -33,34: NewLine (None) -34,35: NL (None) -35,35: Dedent (Indent(1)) -35,35: Dedent (None) -35,40: Identifier (Match) -41,42: Identifier (Str("a")) -42,43: : (None) -43,44: NewLine (None) -44,48: Indent (Indent(1)) -48,52: Identifier (Str("case")) -53,54: Integer (Number("1")) -55,56: | (None) -57,58: Integer (Number("2")) -58,59: : (None) -59,60: NewLine (None) -60,68: Indent (Indent(1)) -68,72: Pass (None) -72,73: NewLine (None) -73,74: NL (None) -74,74: Dedent (Indent(1)) -74,74: Dedent (None) -74,79: Identifier (Match) -80,81: Identifier (Str("a")) -81,82: . (None) -82,83: Identifier (Str("b")) -83,84: : (None) -84,85: NewLine (None) -85,89: Indent (Indent(1)) -89,93: Identifier (Str("case")) -94,95: Integer (Number("1")) -95,96: : (None) -96,97: NewLine (None) -97,105: Indent (Indent(1)) -105,109: Pass (None) -109,110: NewLine (None) -110,111: NL (None) -111,111: Dedent (Indent(1)) -111,111: Dedent (None) -111,116: Identifier (Match) -117,118: Identifier (Str("a")) -118,119: : (None) -119,120: NewLine (None) -120,124: Indent (Indent(1)) -124,128: Identifier (Str("case")) -129,133: None (None) -133,134: : (None) -134,135: NewLine (None) -135,143: Indent (Indent(1)) -143,147: Pass (None) -147,148: NewLine (None) -152,152: Dedent (Indent(1)) -152,156: Identifier (Str("case")) -157,161: True (None) -161,162: : (None) -162,163: NewLine (None) -163,171: Indent (Indent(1)) -171,175: Pass (None) -175,176: NewLine (None) -180,180: Dedent (Indent(1)) -180,184: Identifier (Str("case")) -185,190: False (None) -190,191: : (None) -191,192: NewLine (None) -192,200: Indent (Indent(1)) -200,204: Pass (None) -204,205: NewLine (None) -209,209: Dedent (Indent(1)) -209,213: Identifier (Str("case")) -214,215: - (None) -215,216: Integer (Number("1")) -216,217: : (None) -217,218: NewLine (None) -218,226: Indent (Indent(1)) -226,230: Pass (None) -230,231: NewLine (None) -235,235: Dedent (Indent(1)) -235,239: Identifier (Str("case")) -240,243: PointFloat (Number("1.0")) -243,244: : (None) -244,245: NewLine (None) -245,253: Indent (Indent(1)) -253,257: Pass (None) -257,258: NewLine (None) -262,262: Dedent (Indent(1)) -262,266: Identifier (Str("case")) -267,268: Identifier (Str("_")) -268,269: : (None) -269,270: NewLine (None) -270,278: Indent (Indent(1)) -278,282: Pass (None) -282,283: NewLine (None) -283,284: NL (None) -284,284: Dedent (Indent(1)) -284,284: Dedent (None) -284,289: Identifier (Match) -290,291: Identifier (Str("a")) -291,292: : (None) -292,293: NewLine (None) -293,297: Indent (Indent(1)) -297,301: Identifier (Str("case")) -302,303: Identifier (Str("a")) -303,304: . (None) -304,305: Identifier (Str("b")) -305,306: : (None) -306,307: NewLine (None) -307,315: Indent (Indent(1)) -315,319: Pass (None) -319,320: NewLine (None) -324,324: Dedent (Indent(1)) -324,328: Identifier (Str("case")) -329,330: Identifier (Str("a")) -330,331: : (None) -331,332: NewLine (None) -332,340: Indent (Indent(1)) -340,344: Pass (None) -344,345: NewLine (None) -345,346: NL (None) -346,346: Dedent (Indent(1)) -346,346: Dedent (None) -346,351: Identifier (Match) -352,353: Identifier (Str("a")) -353,354: : (None) -354,355: NewLine (None) -355,359: Indent (Indent(1)) -359,363: Identifier (Str("case")) -364,365: ( (None) -365,366: Identifier (Str("a")) -366,367: , (None) -368,369: Identifier (Str("b")) -369,370: ) (None) -370,371: : (None) -371,372: NewLine (None) -372,380: Indent (Indent(1)) -380,384: Pass (None) -384,385: NewLine (None) -389,389: Dedent (Indent(1)) -389,393: Identifier (Str("case")) -394,395: { (None) -395,396: Integer (Number("1")) -396,397: : (None) -398,399: Identifier (Str("_")) -400,401: , (None) -401,402: Integer (Number("2")) -402,403: : (None) -404,405: Identifier (Str("_")) -405,406: } (None) -406,407: : (None) -407,408: NewLine (None) -408,416: Indent (Indent(1)) -416,420: Pass (None) -420,421: NewLine (None) -425,425: Dedent (Indent(1)) -425,429: Identifier (Str("case")) -430,431: { (None) -431,433: ** (None) -433,437: Identifier (Str("rest")) -437,438: } (None) -438,439: : (None) -439,440: NewLine (None) -440,448: Indent (Indent(1)) -448,452: Pass (None) -452,453: NewLine (None) -453,454: NL (None) -454,454: Dedent (Indent(1)) -454,454: Dedent (None) -454,459: Identifier (Match) -460,461: Identifier (Str("x")) -461,462: : (None) -462,463: NewLine (None) -463,467: Indent (Indent(1)) -467,471: Identifier (Str("case")) -472,479: Identifier (Str("Point2D")) -479,480: ( (None) -480,481: Integer (Number("0")) -481,482: , (None) -483,484: Integer (Number("0")) -484,485: ) (None) -485,486: : (None) -486,487: NewLine (None) -487,495: Indent (Indent(1)) -495,499: Pass (None) -499,500: NewLine (None) -504,504: Dedent (Indent(1)) -504,508: Identifier (Str("case")) -509,516: Identifier (Str("Point3D")) -516,517: ( (None) -517,518: Identifier (Str("x")) -518,519: = (None) -519,520: Integer (Number("0")) -520,521: , (None) -522,523: Identifier (Str("y")) -523,524: = (None) -524,525: Integer (Number("0")) -525,526: , (None) -527,528: Identifier (Str("z")) -528,529: = (None) -529,530: Integer (Number("0")) -530,531: ) (None) -531,532: : (None) -532,533: NewLine (None) -533,541: Indent (Indent(1)) -541,545: Pass (None) -545,546: NewLine (None) -546,547: NL (None) -547,547: Dedent (Indent(1)) -547,547: Dedent (None) -547,552: Identifier (Match) -553,554: Identifier (Str("x")) -554,555: : (None) -555,556: NewLine (None) -556,560: Indent (Indent(1)) -560,564: Identifier (Str("case")) -565,566: [ (None) -566,567: Identifier (Str("a")) -567,568: , (None) -569,570: Identifier (Str("b")) -570,571: , (None) -572,573: Identifier (Str("c")) -573,574: ] (None) -574,575: : (None) -575,576: NewLine (None) -576,584: Indent (Indent(1)) -584,588: Pass (None) -588,589: NewLine (None) -589,589: Dedent (Indent(1)) -589,589: Dedent (None) +0,5: match +6,7: Identifier a +7,8: : +8,9: NewLine +9,13: Indent +13,17: Identifier case +18,19: Integer 1 +19,20: : +20,21: NewLine +21,29: Indent +29,33: Pass +33,34: NewLine +34,35: NL +35,35: Dedent +35,35: Dedent +35,40: match +41,42: Identifier a +42,43: : +43,44: NewLine +44,48: Indent +48,52: Identifier case +53,54: Integer 1 +55,56: | +57,58: Integer 2 +58,59: : +59,60: NewLine +60,68: Indent +68,72: Pass +72,73: NewLine +73,74: NL +74,74: Dedent +74,74: Dedent +74,79: match +80,81: Identifier a +81,82: . +82,83: Identifier b +83,84: : +84,85: NewLine +85,89: Indent +89,93: Identifier case +94,95: Integer 1 +95,96: : +96,97: NewLine +97,105: Indent +105,109: Pass +109,110: NewLine +110,111: NL +111,111: Dedent +111,111: Dedent +111,116: match +117,118: Identifier a +118,119: : +119,120: NewLine +120,124: Indent +124,128: Identifier case +129,133: None +133,134: : +134,135: NewLine +135,143: Indent +143,147: Pass +147,148: NewLine +152,152: Dedent +152,156: Identifier case +157,161: True +161,162: : +162,163: NewLine +163,171: Indent +171,175: Pass +175,176: NewLine +180,180: Dedent +180,184: Identifier case +185,190: False +190,191: : +191,192: NewLine +192,200: Indent +200,204: Pass +204,205: NewLine +209,209: Dedent +209,213: Identifier case +214,215: - +215,216: Integer 1 +216,217: : +217,218: NewLine +218,226: Indent +226,230: Pass +230,231: NewLine +235,235: Dedent +235,239: Identifier case +240,243: PointFloat 1.0 +243,244: : +244,245: NewLine +245,253: Indent +253,257: Pass +257,258: NewLine +262,262: Dedent +262,266: Identifier case +267,268: Identifier _ +268,269: : +269,270: NewLine +270,278: Indent +278,282: Pass +282,283: NewLine +283,284: NL +284,284: Dedent +284,284: Dedent +284,289: match +290,291: Identifier a +291,292: : +292,293: NewLine +293,297: Indent +297,301: Identifier case +302,303: Identifier a +303,304: . +304,305: Identifier b +305,306: : +306,307: NewLine +307,315: Indent +315,319: Pass +319,320: NewLine +324,324: Dedent +324,328: Identifier case +329,330: Identifier a +330,331: : +331,332: NewLine +332,340: Indent +340,344: Pass +344,345: NewLine +345,346: NL +346,346: Dedent +346,346: Dedent +346,351: match +352,353: Identifier a +353,354: : +354,355: NewLine +355,359: Indent +359,363: Identifier case +364,365: ( +365,366: Identifier a +366,367: , +368,369: Identifier b +369,370: ) +370,371: : +371,372: NewLine +372,380: Indent +380,384: Pass +384,385: NewLine +389,389: Dedent +389,393: Identifier case +394,395: { +395,396: Integer 1 +396,397: : +398,399: Identifier _ +400,401: , +401,402: Integer 2 +402,403: : +404,405: Identifier _ +405,406: } +406,407: : +407,408: NewLine +408,416: Indent +416,420: Pass +420,421: NewLine +425,425: Dedent +425,429: Identifier case +430,431: { +431,433: ** +433,437: Identifier rest +437,438: } +438,439: : +439,440: NewLine +440,448: Indent +448,452: Pass +452,453: NewLine +453,454: NL +454,454: Dedent +454,454: Dedent +454,459: match +460,461: Identifier x +461,462: : +462,463: NewLine +463,467: Indent +467,471: Identifier case +472,479: Identifier Point2D +479,480: ( +480,481: Integer 0 +481,482: , +483,484: Integer 0 +484,485: ) +485,486: : +486,487: NewLine +487,495: Indent +495,499: Pass +499,500: NewLine +504,504: Dedent +504,508: Identifier case +509,516: Identifier Point3D +516,517: ( +517,518: Identifier x +518,519: = +519,520: Integer 0 +520,521: , +522,523: Identifier y +523,524: = +524,525: Integer 0 +525,526: , +527,528: Identifier z +528,529: = +529,530: Integer 0 +530,531: ) +531,532: : +532,533: NewLine +533,541: Indent +541,545: Pass +545,546: NewLine +546,547: NL +547,547: Dedent +547,547: Dedent +547,552: match +553,554: Identifier x +554,555: : +555,556: NewLine +556,560: Indent +560,564: Identifier case +565,566: [ +566,567: Identifier a +567,568: , +569,570: Identifier b +570,571: , +572,573: Identifier c +573,574: ] +574,575: : +575,576: NewLine +576,584: Indent +584,588: Pass +588,589: NewLine +589,589: Dedent +589,589: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap index 7af5b664..83bceef9 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@newlines.py.snap @@ -3,71 +3,71 @@ source: parser/src/lexer/mod.rs description: "_ = (\n 1,\n 2,\n)\n_ = {\n 1,\n 2,\n}\n_ = [\n 1,\n 2,\n]\n\n\ndef hello(\n a,\n b,\n): ...\n\n\nclass A(\n object,\n):\n pass\n" input_file: parser/test_data/inputs/newlines.py --- -0,1: Identifier (Str("_")) -2,3: = (None) -4,5: ( (None) -5,6: NL (None) -10,11: Integer (Number("1")) -11,12: , (None) -12,13: NL (None) -17,18: Integer (Number("2")) -18,19: , (None) -19,20: NL (None) -20,21: ) (None) -21,22: NewLine (None) -22,23: Identifier (Str("_")) -24,25: = (None) -26,27: { (None) -27,28: NL (None) -32,33: Integer (Number("1")) -33,34: , (None) -34,35: NL (None) -39,40: Integer (Number("2")) -40,41: , (None) -41,42: NL (None) -42,43: } (None) -43,44: NewLine (None) -44,45: Identifier (Str("_")) -46,47: = (None) -48,49: [ (None) -49,50: NL (None) -54,55: Integer (Number("1")) -55,56: , (None) -56,57: NL (None) -61,62: Integer (Number("2")) -62,63: , (None) -63,64: NL (None) -64,65: ] (None) -65,66: NewLine (None) -66,67: NL (None) -67,68: NL (None) -68,71: Def (None) -72,77: Identifier (Str("hello")) -77,78: ( (None) -78,79: NL (None) -83,84: Identifier (Str("a")) -84,85: , (None) -85,86: NL (None) -90,91: Identifier (Str("b")) -91,92: , (None) -92,93: NL (None) -93,94: ) (None) -94,95: : (None) -96,99: Ellipsis (None) -99,100: NewLine (None) -100,101: NL (None) -101,102: NL (None) -102,107: Class (None) -108,109: Identifier (Str("A")) -109,110: ( (None) -110,111: NL (None) -115,121: Identifier (Str("object")) -121,122: , (None) -122,123: NL (None) -123,124: ) (None) -124,125: : (None) -125,126: NewLine (None) -126,130: Indent (Indent(1)) -130,134: Pass (None) -134,135: NewLine (None) -135,135: Dedent (Indent(1)) +0,1: Identifier _ +2,3: = +4,5: ( +5,6: NL +10,11: Integer 1 +11,12: , +12,13: NL +17,18: Integer 2 +18,19: , +19,20: NL +20,21: ) +21,22: NewLine +22,23: Identifier _ +24,25: = +26,27: { +27,28: NL +32,33: Integer 1 +33,34: , +34,35: NL +39,40: Integer 2 +40,41: , +41,42: NL +42,43: } +43,44: NewLine +44,45: Identifier _ +46,47: = +48,49: [ +49,50: NL +54,55: Integer 1 +55,56: , +56,57: NL +61,62: Integer 2 +62,63: , +63,64: NL +64,65: ] +65,66: NewLine +66,67: NL +67,68: NL +68,71: Def +72,77: Identifier hello +77,78: ( +78,79: NL +83,84: Identifier a +84,85: , +85,86: NL +90,91: Identifier b +91,92: , +92,93: NL +93,94: ) +94,95: : +96,99: Ellipsis +99,100: NewLine +100,101: NL +101,102: NL +102,107: Class +108,109: Identifier A +109,110: ( +110,111: NL +115,121: Identifier object +121,122: , +122,123: NL +123,124: ) +124,125: : +125,126: NewLine +126,130: Indent +130,134: Pass +134,135: NewLine +135,135: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap index abf939ab..b0ffcc94 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@separate_statements.py.snap @@ -3,50 +3,50 @@ source: parser/src/lexer/mod.rs description: "# Test case to check that we return correct error when two python statements are on the same line\n\ndef foo():\n x = 1; y = 2\n return x + y\n\ndef bar():\n # TODO(parser): enable after error handling\n # x = 1 y = 2\n return x + y\n\n# TODO(parser): enable after error handling\n# a = 1 b = 2\n" input_file: parser/test_data/inputs/separate_statements.py --- -0,97: Comment (Str("# Test case to check that we return correct error when two python statements are on the same line")) -97,98: NL (None) -98,99: NL (None) -99,102: Def (None) -103,106: Identifier (Str("foo")) -106,107: ( (None) -107,108: ) (None) -108,109: : (None) -109,110: NewLine (None) -110,114: Indent (Indent(1)) -114,115: Identifier (Str("x")) -116,117: = (None) -118,119: Integer (Number("1")) -119,120: ; (None) -121,122: Identifier (Str("y")) -123,124: = (None) -125,126: Integer (Number("2")) -126,127: NewLine (None) -131,137: Return (None) -138,139: Identifier (Str("x")) -140,141: + (None) -142,143: Identifier (Str("y")) -143,144: NewLine (None) -144,145: NL (None) -145,145: Dedent (Indent(1)) -145,148: Def (None) -149,152: Identifier (Str("bar")) -152,153: ( (None) -153,154: ) (None) -154,155: : (None) -155,156: NewLine (None) -160,203: Comment (Str("# TODO(parser): enable after error handling")) -203,204: NL (None) -208,221: Comment (Str("# x = 1 y = 2")) -221,222: NL (None) -222,226: Indent (Indent(1)) -226,232: Return (None) -233,234: Identifier (Str("x")) -235,236: + (None) -237,238: Identifier (Str("y")) -238,239: NewLine (None) -239,240: NL (None) -240,283: Comment (Str("# TODO(parser): enable after error handling")) -283,284: NL (None) -284,298: Comment (Str("# a = 1 b = 2")) -298,299: NL (None) -299,299: Dedent (Indent(1)) +0,97: Comment +97,98: NL +98,99: NL +99,102: Def +103,106: Identifier foo +106,107: ( +107,108: ) +108,109: : +109,110: NewLine +110,114: Indent +114,115: Identifier x +116,117: = +118,119: Integer 1 +119,120: ; +121,122: Identifier y +123,124: = +125,126: Integer 2 +126,127: NewLine +131,137: Return +138,139: Identifier x +140,141: + +142,143: Identifier y +143,144: NewLine +144,145: NL +145,145: Dedent +145,148: Def +149,152: Identifier bar +152,153: ( +153,154: ) +154,155: : +155,156: NewLine +160,203: Comment +203,204: NL +208,221: Comment +221,222: NL +222,226: Indent +226,232: Return +233,234: Identifier x +235,236: + +237,238: Identifier y +238,239: NewLine +239,240: NL +240,283: Comment +283,284: NL +284,298: Comment +298,299: NL +299,299: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@sets.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@sets.py.snap index 2ac637a2..45b950d0 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@sets.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@sets.py.snap @@ -3,192 +3,192 @@ source: parser/src/lexer/mod.rs description: "{a, b, c}\n\n{a,\nb, c}\n\n{a\n, b, c}\n\n{a,\nb,\n c}\n\n{a,\n}\n\n{a, b, c,}\n\n{a for a in range(10)}\n\n{a for a in range(10)\n}\n\n{a for a in range(10)}\n\n{a for a in range(10)}\n\n{a for a in range(10) for b in range(10)}\n\n{a for a in range(10) for b in range(10)\n}\n\n{a for a in range(11) for b in range(10)\n}\n\n{\n (a,b) for a in range(11) for b in range(10)\n}\n" input_file: parser/test_data/inputs/sets.py --- -0,1: { (None) -1,2: Identifier (Str("a")) -2,3: , (None) -4,5: Identifier (Str("b")) -5,6: , (None) -7,8: Identifier (Str("c")) -8,9: } (None) -9,10: NewLine (None) -10,11: NL (None) -11,12: { (None) -12,13: Identifier (Str("a")) -13,14: , (None) -14,15: NL (None) -15,16: Identifier (Str("b")) -16,17: , (None) -18,19: Identifier (Str("c")) -19,20: } (None) -20,21: NewLine (None) -21,22: NL (None) -22,23: { (None) -23,24: Identifier (Str("a")) -24,25: NL (None) -25,26: , (None) -27,28: Identifier (Str("b")) -28,29: , (None) -30,31: Identifier (Str("c")) -31,32: } (None) -32,33: NewLine (None) -33,34: NL (None) -34,35: { (None) -35,36: Identifier (Str("a")) -36,37: , (None) -37,38: NL (None) -38,39: Identifier (Str("b")) -39,40: , (None) -40,41: NL (None) -45,46: Identifier (Str("c")) -46,47: } (None) -47,48: NewLine (None) -48,49: NL (None) -49,50: { (None) -50,51: Identifier (Str("a")) -51,52: , (None) -52,53: NL (None) -53,54: } (None) -54,55: NewLine (None) -55,56: NL (None) -56,57: { (None) -57,58: Identifier (Str("a")) -58,59: , (None) -60,61: Identifier (Str("b")) -61,62: , (None) -63,64: Identifier (Str("c")) -64,65: , (None) -65,66: } (None) -66,67: NewLine (None) -67,68: NL (None) -68,69: { (None) -69,70: Identifier (Str("a")) -71,74: For (None) -75,76: Identifier (Str("a")) -77,79: In (None) -80,85: Identifier (Str("range")) -85,86: ( (None) -86,88: Integer (Number("10")) -88,89: ) (None) -89,90: } (None) -90,91: NewLine (None) -91,92: NL (None) -92,93: { (None) -93,94: Identifier (Str("a")) -95,98: For (None) -99,100: Identifier (Str("a")) -101,103: In (None) -104,109: Identifier (Str("range")) -109,110: ( (None) -110,112: Integer (Number("10")) -112,113: ) (None) -113,114: NL (None) -114,115: } (None) -115,116: NewLine (None) -116,117: NL (None) -117,118: { (None) -118,119: Identifier (Str("a")) -120,123: For (None) -124,125: Identifier (Str("a")) -126,128: In (None) -129,134: Identifier (Str("range")) -134,135: ( (None) -135,137: Integer (Number("10")) -137,138: ) (None) -138,139: } (None) -139,140: NewLine (None) -140,141: NL (None) -141,142: { (None) -142,143: Identifier (Str("a")) -144,147: For (None) -148,149: Identifier (Str("a")) -150,152: In (None) -153,158: Identifier (Str("range")) -158,159: ( (None) -159,161: Integer (Number("10")) -161,162: ) (None) -162,163: } (None) -163,164: NewLine (None) -164,165: NL (None) -165,166: { (None) -166,167: Identifier (Str("a")) -168,171: For (None) -172,173: Identifier (Str("a")) -174,176: In (None) -177,182: Identifier (Str("range")) -182,183: ( (None) -183,185: Integer (Number("10")) -185,186: ) (None) -187,190: For (None) -191,192: Identifier (Str("b")) -193,195: In (None) -196,201: Identifier (Str("range")) -201,202: ( (None) -202,204: Integer (Number("10")) -204,205: ) (None) -205,206: } (None) -206,207: NewLine (None) -207,208: NL (None) -208,209: { (None) -209,210: Identifier (Str("a")) -211,214: For (None) -215,216: Identifier (Str("a")) -217,219: In (None) -220,225: Identifier (Str("range")) -225,226: ( (None) -226,228: Integer (Number("10")) -228,229: ) (None) -230,233: For (None) -234,235: Identifier (Str("b")) -236,238: In (None) -239,244: Identifier (Str("range")) -244,245: ( (None) -245,247: Integer (Number("10")) -247,248: ) (None) -248,249: NL (None) -249,250: } (None) -250,251: NewLine (None) -251,252: NL (None) -252,253: { (None) -253,254: Identifier (Str("a")) -255,258: For (None) -259,260: Identifier (Str("a")) -261,263: In (None) -264,269: Identifier (Str("range")) -269,270: ( (None) -270,272: Integer (Number("11")) -272,273: ) (None) -274,277: For (None) -278,279: Identifier (Str("b")) -280,282: In (None) -283,288: Identifier (Str("range")) -288,289: ( (None) -289,291: Integer (Number("10")) -291,292: ) (None) -292,293: NL (None) -293,294: } (None) -294,295: NewLine (None) -295,296: NL (None) -296,297: { (None) -297,298: NL (None) -302,303: ( (None) -303,304: Identifier (Str("a")) -304,305: , (None) -305,306: Identifier (Str("b")) -306,307: ) (None) -308,311: For (None) -312,313: Identifier (Str("a")) -314,316: In (None) -317,322: Identifier (Str("range")) -322,323: ( (None) -323,325: Integer (Number("11")) -325,326: ) (None) -327,330: For (None) -331,332: Identifier (Str("b")) -333,335: In (None) -336,341: Identifier (Str("range")) -341,342: ( (None) -342,344: Integer (Number("10")) -344,345: ) (None) -345,346: NL (None) -346,347: } (None) -347,348: NewLine (None) +0,1: { +1,2: Identifier a +2,3: , +4,5: Identifier b +5,6: , +7,8: Identifier c +8,9: } +9,10: NewLine +10,11: NL +11,12: { +12,13: Identifier a +13,14: , +14,15: NL +15,16: Identifier b +16,17: , +18,19: Identifier c +19,20: } +20,21: NewLine +21,22: NL +22,23: { +23,24: Identifier a +24,25: NL +25,26: , +27,28: Identifier b +28,29: , +30,31: Identifier c +31,32: } +32,33: NewLine +33,34: NL +34,35: { +35,36: Identifier a +36,37: , +37,38: NL +38,39: Identifier b +39,40: , +40,41: NL +45,46: Identifier c +46,47: } +47,48: NewLine +48,49: NL +49,50: { +50,51: Identifier a +51,52: , +52,53: NL +53,54: } +54,55: NewLine +55,56: NL +56,57: { +57,58: Identifier a +58,59: , +60,61: Identifier b +61,62: , +63,64: Identifier c +64,65: , +65,66: } +66,67: NewLine +67,68: NL +68,69: { +69,70: Identifier a +71,74: For +75,76: Identifier a +77,79: In +80,85: Identifier range +85,86: ( +86,88: Integer 10 +88,89: ) +89,90: } +90,91: NewLine +91,92: NL +92,93: { +93,94: Identifier a +95,98: For +99,100: Identifier a +101,103: In +104,109: Identifier range +109,110: ( +110,112: Integer 10 +112,113: ) +113,114: NL +114,115: } +115,116: NewLine +116,117: NL +117,118: { +118,119: Identifier a +120,123: For +124,125: Identifier a +126,128: In +129,134: Identifier range +134,135: ( +135,137: Integer 10 +137,138: ) +138,139: } +139,140: NewLine +140,141: NL +141,142: { +142,143: Identifier a +144,147: For +148,149: Identifier a +150,152: In +153,158: Identifier range +158,159: ( +159,161: Integer 10 +161,162: ) +162,163: } +163,164: NewLine +164,165: NL +165,166: { +166,167: Identifier a +168,171: For +172,173: Identifier a +174,176: In +177,182: Identifier range +182,183: ( +183,185: Integer 10 +185,186: ) +187,190: For +191,192: Identifier b +193,195: In +196,201: Identifier range +201,202: ( +202,204: Integer 10 +204,205: ) +205,206: } +206,207: NewLine +207,208: NL +208,209: { +209,210: Identifier a +211,214: For +215,216: Identifier a +217,219: In +220,225: Identifier range +225,226: ( +226,228: Integer 10 +228,229: ) +230,233: For +234,235: Identifier b +236,238: In +239,244: Identifier range +244,245: ( +245,247: Integer 10 +247,248: ) +248,249: NL +249,250: } +250,251: NewLine +251,252: NL +252,253: { +253,254: Identifier a +255,258: For +259,260: Identifier a +261,263: In +264,269: Identifier range +269,270: ( +270,272: Integer 11 +272,273: ) +274,277: For +278,279: Identifier b +280,282: In +283,288: Identifier range +288,289: ( +289,291: Integer 10 +291,292: ) +292,293: NL +293,294: } +294,295: NewLine +295,296: NL +296,297: { +297,298: NL +302,303: ( +303,304: Identifier a +304,305: , +305,306: Identifier b +306,307: ) +308,311: For +312,313: Identifier a +314,316: In +317,322: Identifier range +322,323: ( +323,325: Integer 11 +325,326: ) +327,330: For +331,332: Identifier b +333,335: In +336,341: Identifier range +341,342: ( +342,344: Integer 10 +344,345: ) +345,346: NL +346,347: } +347,348: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap index 9aeb2be6..9780832f 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap @@ -3,133 +3,133 @@ source: parser/src/lexer/mod.rs description: "x = \"simple text\\n\"\n\nraw = r\"raw_text\\n\"\n\ny = \"\"\"multi\"\"\"\n\nf\"{self.__class__.__name__}({self._display()!r})\"\n\nf\"{num:0.0f}{unit}\"\n\nf\"tuple argument {name[12:]}\"\n\nf\"some words {a+b:.3f} more words {c+d=} final words\"\n\nf\"{f\"{f\"{f\"{f\"{f\"{1+1}\"}\"}\"}\"}\"}\"\n\nf\"\"\"{f'''{f'{f\"{1+1}\"}'}'''}\"\"\"\n" input_file: parser/test_data/inputs/string.py --- -0,1: Identifier (Str("x")) -2,3: = (None) -4,19: StringLiteral (Str("\"simple text\\n\"")) -19,20: NewLine (None) -20,21: NL (None) -21,24: Identifier (Str("raw")) -25,26: = (None) -27,40: StringLiteral (Str("r\"raw_text\\n\"")) -40,41: NewLine (None) -41,42: NL (None) -42,43: Identifier (Str("y")) -44,45: = (None) -46,57: StringLiteral (Str("\"\"\"multi\"\"\"")) -57,58: NewLine (None) -58,59: NL (None) -59,61: FStringStart (Str("f\"")) -61,62: { (None) -62,66: Identifier (Str("self")) -66,67: . (None) -67,76: Identifier (Str("__class__")) -76,77: . (None) -77,85: Identifier (Str("__name__")) -85,86: } (None) -86,87: FstringMiddle (Str("(")) -87,88: { (None) -88,92: Identifier (Str("self")) -92,93: . (None) -93,101: Identifier (Str("_display")) -101,102: ( (None) -102,103: ) (None) -103,104: ! (None) -104,105: Identifier (Str("r")) -105,106: } (None) -106,107: FstringMiddle (Str(")")) -107,108: FStringEnd (Str("\"")) -108,109: NewLine (None) -109,110: NL (None) -110,112: FStringStart (Str("f\"")) -112,113: { (None) -113,116: Identifier (Str("num")) -116,117: : (None) -117,121: FstringMiddle (Str("0.0f")) -121,122: } (None) -122,123: { (None) -123,127: Identifier (Str("unit")) -127,128: } (None) -128,129: FStringEnd (Str("\"")) -129,130: NewLine (None) -130,131: NL (None) -131,133: FStringStart (Str("f\"")) -133,148: FstringMiddle (Str("tuple argument ")) -148,149: { (None) -149,153: Identifier (Str("name")) -153,154: [ (None) -154,156: Integer (Number("12")) -156,157: : (None) -157,158: ] (None) -158,159: } (None) -159,160: FStringEnd (Str("\"")) -160,161: NewLine (None) -161,162: NL (None) -162,164: FStringStart (Str("f\"")) -164,175: FstringMiddle (Str("some words ")) -175,176: { (None) -176,177: Identifier (Str("a")) -177,178: + (None) -178,179: Identifier (Str("b")) -179,180: : (None) -180,183: FstringMiddle (Str(".3f")) -183,184: } (None) -184,196: FstringMiddle (Str(" more words ")) -196,197: { (None) -197,198: Identifier (Str("c")) -198,199: + (None) -199,200: Identifier (Str("d")) -200,201: = (None) -201,202: } (None) -202,214: FstringMiddle (Str(" final words")) -214,215: FStringEnd (Str("\"")) -215,216: NewLine (None) -216,217: NL (None) -217,219: FStringStart (Str("f\"")) -219,220: { (None) -220,222: FStringStart (Str("f\"")) -222,223: { (None) -223,225: FStringStart (Str("f\"")) -225,226: { (None) -226,228: FStringStart (Str("f\"")) -228,229: { (None) -229,231: FStringStart (Str("f\"")) -231,232: { (None) -232,234: FStringStart (Str("f\"")) -234,235: { (None) -235,236: Integer (Number("1")) -236,237: + (None) -237,238: Integer (Number("1")) -238,239: } (None) -239,240: FStringEnd (Str("\"")) -240,241: } (None) -241,242: FStringEnd (Str("\"")) -242,243: } (None) -243,244: FStringEnd (Str("\"")) -244,245: } (None) -245,246: FStringEnd (Str("\"")) -246,247: } (None) -247,248: FStringEnd (Str("\"")) -248,249: } (None) -249,250: FStringEnd (Str("\"")) -250,251: NewLine (None) -251,252: NL (None) -252,256: FStringStart (Str("f\"\"\"")) -256,257: { (None) -257,261: FStringStart (Str("f'''")) -261,262: { (None) -262,264: FStringStart (Str("f'")) -264,265: { (None) -265,267: FStringStart (Str("f\"")) -267,268: { (None) -268,269: Integer (Number("1")) -269,270: + (None) -270,271: Integer (Number("1")) -271,272: } (None) -272,273: FStringEnd (Str("\"")) -273,274: } (None) -274,275: FStringEnd (Str("'")) -275,276: } (None) -276,279: FStringEnd (Str("'''")) -279,280: } (None) -280,283: FStringEnd (Str("\"\"\"")) -283,284: NewLine (None) +0,1: Identifier x +2,3: = +4,19: StringLiteral "simple text\n" +19,20: NewLine +20,21: NL +21,24: Identifier raw +25,26: = +27,40: StringLiteral r"raw_text\n" +40,41: NewLine +41,42: NL +42,43: Identifier y +44,45: = +46,57: StringLiteral """multi""" +57,58: NewLine +58,59: NL +59,61: FStringStart +61,62: { +62,66: Identifier self +66,67: . +67,76: Identifier __class__ +76,77: . +77,85: Identifier __name__ +85,86: } +86,87: FstringMiddle ( +87,88: { +88,92: Identifier self +92,93: . +93,101: Identifier _display +101,102: ( +102,103: ) +103,104: ! +104,105: Identifier r +105,106: } +106,107: FstringMiddle ) +107,108: FStringEnd +108,109: NewLine +109,110: NL +110,112: FStringStart +112,113: { +113,116: Identifier num +116,117: : +117,121: FstringMiddle 0.0f +121,122: } +122,123: { +123,127: Identifier unit +127,128: } +128,129: FStringEnd +129,130: NewLine +130,131: NL +131,133: FStringStart +133,148: FstringMiddle tuple argument +148,149: { +149,153: Identifier name +153,154: [ +154,156: Integer 12 +156,157: : +157,158: ] +158,159: } +159,160: FStringEnd +160,161: NewLine +161,162: NL +162,164: FStringStart +164,175: FstringMiddle some words +175,176: { +176,177: Identifier a +177,178: + +178,179: Identifier b +179,180: : +180,183: FstringMiddle .3f +183,184: } +184,196: FstringMiddle more words +196,197: { +197,198: Identifier c +198,199: + +199,200: Identifier d +200,201: = +201,202: } +202,214: FstringMiddle final words +214,215: FStringEnd +215,216: NewLine +216,217: NL +217,219: FStringStart +219,220: { +220,222: FStringStart +222,223: { +223,225: FStringStart +225,226: { +226,228: FStringStart +228,229: { +229,231: FStringStart +231,232: { +232,234: FStringStart +234,235: { +235,236: Integer 1 +236,237: + +237,238: Integer 1 +238,239: } +239,240: FStringEnd +240,241: } +241,242: FStringEnd +242,243: } +243,244: FStringEnd +244,245: } +245,246: FStringEnd +246,247: } +247,248: FStringEnd +248,249: } +249,250: FStringEnd +250,251: NewLine +251,252: NL +252,256: FStringStart +256,257: { +257,261: FStringStart +261,262: { +262,264: FStringStart +264,265: { +265,267: FStringStart +267,268: { +268,269: Integer 1 +269,270: + +270,271: Integer 1 +271,272: } +272,273: FStringEnd +273,274: } +274,275: FStringEnd +275,276: } +276,279: FStringEnd +279,280: } +280,283: FStringEnd +283,284: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@subscript.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@subscript.py.snap index 0f3b6a3d..c88213bd 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@subscript.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@subscript.py.snap @@ -3,156 +3,156 @@ source: parser/src/lexer/mod.rs description: "indices = list_of_key_stats[0::2]\n\nvalues = list_of_key_stats[1::2]\n\na[b]\n\na[::]\na[::step]\na[::step,]\n\na[:upper]\na[:6]\na[:upper:]\na[:upper:step]\n\na[lower:]\na[lower:upper]\na[lower:upper:]\na[lower:upper:step]\na[lower::]\n\n\na[b, c, d]\n\na[b, c:d, e]\n\n\na[b, c:d:e, f]\n" input_file: parser/test_data/inputs/subscript.py --- -0,7: Identifier (Str("indices")) -8,9: = (None) -10,27: Identifier (Str("list_of_key_stats")) -27,28: [ (None) -28,29: Integer (Number("0")) -29,30: : (None) -30,31: : (None) -31,32: Integer (Number("2")) -32,33: ] (None) -33,34: NewLine (None) -34,35: NL (None) -35,41: Identifier (Str("values")) -42,43: = (None) -44,61: Identifier (Str("list_of_key_stats")) -61,62: [ (None) -62,63: Integer (Number("1")) -63,64: : (None) -64,65: : (None) -65,66: Integer (Number("2")) -66,67: ] (None) -67,68: NewLine (None) -68,69: NL (None) -69,70: Identifier (Str("a")) -70,71: [ (None) -71,72: Identifier (Str("b")) -72,73: ] (None) -73,74: NewLine (None) -74,75: NL (None) -75,76: Identifier (Str("a")) -76,77: [ (None) -77,78: : (None) -78,79: : (None) -79,80: ] (None) -80,81: NewLine (None) -81,82: Identifier (Str("a")) -82,83: [ (None) -83,84: : (None) -84,85: : (None) -85,89: Identifier (Str("step")) -89,90: ] (None) -90,91: NewLine (None) -91,92: Identifier (Str("a")) -92,93: [ (None) -93,94: : (None) -94,95: : (None) -95,99: Identifier (Str("step")) -99,100: , (None) -100,101: ] (None) -101,102: NewLine (None) -102,103: NL (None) -103,104: Identifier (Str("a")) -104,105: [ (None) -105,106: : (None) -106,111: Identifier (Str("upper")) -111,112: ] (None) -112,113: NewLine (None) -113,114: Identifier (Str("a")) -114,115: [ (None) -115,116: : (None) -116,117: Integer (Number("6")) -117,118: ] (None) -118,119: NewLine (None) -119,120: Identifier (Str("a")) -120,121: [ (None) -121,122: : (None) -122,127: Identifier (Str("upper")) -127,128: : (None) -128,129: ] (None) -129,130: NewLine (None) -130,131: Identifier (Str("a")) -131,132: [ (None) -132,133: : (None) -133,138: Identifier (Str("upper")) -138,139: : (None) -139,143: Identifier (Str("step")) -143,144: ] (None) -144,145: NewLine (None) -145,146: NL (None) -146,147: Identifier (Str("a")) -147,148: [ (None) -148,153: Identifier (Str("lower")) -153,154: : (None) -154,155: ] (None) -155,156: NewLine (None) -156,157: Identifier (Str("a")) -157,158: [ (None) -158,163: Identifier (Str("lower")) -163,164: : (None) -164,169: Identifier (Str("upper")) -169,170: ] (None) -170,171: NewLine (None) -171,172: Identifier (Str("a")) -172,173: [ (None) -173,178: Identifier (Str("lower")) -178,179: : (None) -179,184: Identifier (Str("upper")) -184,185: : (None) -185,186: ] (None) -186,187: NewLine (None) -187,188: Identifier (Str("a")) -188,189: [ (None) -189,194: Identifier (Str("lower")) -194,195: : (None) -195,200: Identifier (Str("upper")) -200,201: : (None) -201,205: Identifier (Str("step")) -205,206: ] (None) -206,207: NewLine (None) -207,208: Identifier (Str("a")) -208,209: [ (None) -209,214: Identifier (Str("lower")) -214,215: : (None) -215,216: : (None) -216,217: ] (None) -217,218: NewLine (None) -218,219: NL (None) -219,220: NL (None) -220,221: Identifier (Str("a")) -221,222: [ (None) -222,223: Identifier (Str("b")) -223,224: , (None) -225,226: Identifier (Str("c")) -226,227: , (None) -228,229: Identifier (Str("d")) -229,230: ] (None) -230,231: NewLine (None) -231,232: NL (None) -232,233: Identifier (Str("a")) -233,234: [ (None) -234,235: Identifier (Str("b")) -235,236: , (None) -237,238: Identifier (Str("c")) -238,239: : (None) -239,240: Identifier (Str("d")) -240,241: , (None) -242,243: Identifier (Str("e")) -243,244: ] (None) -244,245: NewLine (None) -245,246: NL (None) -246,247: NL (None) -247,248: Identifier (Str("a")) -248,249: [ (None) -249,250: Identifier (Str("b")) -250,251: , (None) -252,253: Identifier (Str("c")) -253,254: : (None) -254,255: Identifier (Str("d")) -255,256: : (None) -256,257: Identifier (Str("e")) -257,258: , (None) -259,260: Identifier (Str("f")) -260,261: ] (None) -261,262: NewLine (None) +0,7: Identifier indices +8,9: = +10,27: Identifier list_of_key_stats +27,28: [ +28,29: Integer 0 +29,30: : +30,31: : +31,32: Integer 2 +32,33: ] +33,34: NewLine +34,35: NL +35,41: Identifier values +42,43: = +44,61: Identifier list_of_key_stats +61,62: [ +62,63: Integer 1 +63,64: : +64,65: : +65,66: Integer 2 +66,67: ] +67,68: NewLine +68,69: NL +69,70: Identifier a +70,71: [ +71,72: Identifier b +72,73: ] +73,74: NewLine +74,75: NL +75,76: Identifier a +76,77: [ +77,78: : +78,79: : +79,80: ] +80,81: NewLine +81,82: Identifier a +82,83: [ +83,84: : +84,85: : +85,89: Identifier step +89,90: ] +90,91: NewLine +91,92: Identifier a +92,93: [ +93,94: : +94,95: : +95,99: Identifier step +99,100: , +100,101: ] +101,102: NewLine +102,103: NL +103,104: Identifier a +104,105: [ +105,106: : +106,111: Identifier upper +111,112: ] +112,113: NewLine +113,114: Identifier a +114,115: [ +115,116: : +116,117: Integer 6 +117,118: ] +118,119: NewLine +119,120: Identifier a +120,121: [ +121,122: : +122,127: Identifier upper +127,128: : +128,129: ] +129,130: NewLine +130,131: Identifier a +131,132: [ +132,133: : +133,138: Identifier upper +138,139: : +139,143: Identifier step +143,144: ] +144,145: NewLine +145,146: NL +146,147: Identifier a +147,148: [ +148,153: Identifier lower +153,154: : +154,155: ] +155,156: NewLine +156,157: Identifier a +157,158: [ +158,163: Identifier lower +163,164: : +164,169: Identifier upper +169,170: ] +170,171: NewLine +171,172: Identifier a +172,173: [ +173,178: Identifier lower +178,179: : +179,184: Identifier upper +184,185: : +185,186: ] +186,187: NewLine +187,188: Identifier a +188,189: [ +189,194: Identifier lower +194,195: : +195,200: Identifier upper +200,201: : +201,205: Identifier step +205,206: ] +206,207: NewLine +207,208: Identifier a +208,209: [ +209,214: Identifier lower +214,215: : +215,216: : +216,217: ] +217,218: NewLine +218,219: NL +219,220: NL +220,221: Identifier a +221,222: [ +222,223: Identifier b +223,224: , +225,226: Identifier c +226,227: , +228,229: Identifier d +229,230: ] +230,231: NewLine +231,232: NL +232,233: Identifier a +233,234: [ +234,235: Identifier b +235,236: , +237,238: Identifier c +238,239: : +239,240: Identifier d +240,241: , +242,243: Identifier e +243,244: ] +244,245: NewLine +245,246: NL +246,247: NL +247,248: Identifier a +248,249: [ +249,250: Identifier b +250,251: , +252,253: Identifier c +253,254: : +254,255: Identifier d +255,256: : +256,257: Identifier e +257,258: , +259,260: Identifier f +260,261: ] +261,262: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap index 457ab2f2..18468b2d 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap @@ -3,239 +3,239 @@ source: parser/src/lexer/mod.rs description: "# in some cases last price or adj price is undefined\ntry:\n last_price = int(price_section[2])\n# when instead of number value is `F`\nexcept (ValueError, IndexError):\n last_price = None\ntry:\n adj_close = int(price_section[3])\nexcept (ValueError, IndexError):\n adj_close = None\ntry:\n market_cap = adj_close * self.total_shares\nexcept ValueError:\n market_cap = None\n\n\ntry:\n async with session.get(url, headers=TRADE_DETAILS_HEADER, timeout=100) as response:\n if response.status == 503:\n logger.info(f\"Received 503 Service Unavailable on {date_obj}. Retrying...\")\n retry_count += 1\n await asyncio.sleep(1)\n else:\n response.raise_for_status()\n data = await response.json()\n logger.info(f\"Successfully fetched trade details on {date_obj} from tse\")\n return [date_obj, pd.json_normalize(data[\"tradeHistory\"])]\nexcept (aiohttp.ClientError, asyncio.TimeoutError):\n logger.error(f\"Request failed for {date_obj}. Retrying...\")\n retry_count += 1\n await asyncio.sleep(1)\n" input_file: parser/test_data/inputs/try.py --- -0,52: Comment (Str("# in some cases last price or adj price is undefined")) -52,53: NL (None) -53,56: Try (None) -56,57: : (None) -57,58: NewLine (None) -58,62: Indent (Indent(1)) -62,72: Identifier (Str("last_price")) -73,74: = (None) -75,78: Identifier (Str("int")) -78,79: ( (None) -79,92: Identifier (Str("price_section")) -92,93: [ (None) -93,94: Integer (Number("2")) -94,95: ] (None) -95,96: ) (None) -96,97: NewLine (None) -97,134: Comment (Str("# when instead of number value is `F`")) -134,135: NL (None) -135,135: Dedent (Indent(1)) -135,141: Except (None) -142,143: ( (None) -143,153: Identifier (Str("ValueError")) -153,154: , (None) -155,165: Identifier (Str("IndexError")) -165,166: ) (None) -166,167: : (None) -167,168: NewLine (None) -168,172: Indent (Indent(1)) -172,182: Identifier (Str("last_price")) -183,184: = (None) -185,189: None (None) -189,190: NewLine (None) -190,190: Dedent (Indent(1)) -190,193: Try (None) -193,194: : (None) -194,195: NewLine (None) -195,199: Indent (Indent(1)) -199,208: Identifier (Str("adj_close")) -209,210: = (None) -211,214: Identifier (Str("int")) -214,215: ( (None) -215,228: Identifier (Str("price_section")) -228,229: [ (None) -229,230: Integer (Number("3")) -230,231: ] (None) -231,232: ) (None) -232,233: NewLine (None) -233,233: Dedent (Indent(1)) -233,239: Except (None) -240,241: ( (None) -241,251: Identifier (Str("ValueError")) -251,252: , (None) -253,263: Identifier (Str("IndexError")) -263,264: ) (None) -264,265: : (None) -265,266: NewLine (None) -266,270: Indent (Indent(1)) -270,279: Identifier (Str("adj_close")) -280,281: = (None) -282,286: None (None) -286,287: NewLine (None) -287,287: Dedent (Indent(1)) -287,290: Try (None) -290,291: : (None) -291,292: NewLine (None) -292,296: Indent (Indent(1)) -296,306: Identifier (Str("market_cap")) -307,308: = (None) -309,318: Identifier (Str("adj_close")) -319,320: * (None) -321,325: Identifier (Str("self")) -325,326: . (None) -326,338: Identifier (Str("total_shares")) -338,339: NewLine (None) -339,339: Dedent (Indent(1)) -339,345: Except (None) -346,356: Identifier (Str("ValueError")) -356,357: : (None) -357,358: NewLine (None) -358,362: Indent (Indent(1)) -362,372: Identifier (Str("market_cap")) -373,374: = (None) -375,379: None (None) -379,380: NewLine (None) -380,381: NL (None) -381,382: NL (None) -382,382: Dedent (Indent(1)) -382,385: Try (None) -385,386: : (None) -386,387: NewLine (None) -387,391: Indent (Indent(1)) -391,396: Async (None) -397,401: With (None) -402,409: Identifier (Str("session")) -409,410: . (None) -410,413: Identifier (Str("get")) -413,414: ( (None) -414,417: Identifier (Str("url")) -417,418: , (None) -419,426: Identifier (Str("headers")) -426,427: = (None) -427,447: Identifier (Str("TRADE_DETAILS_HEADER")) -447,448: , (None) -449,456: Identifier (Str("timeout")) -456,457: = (None) -457,460: Integer (Number("100")) -460,461: ) (None) -462,464: As (None) -465,473: Identifier (Str("response")) -473,474: : (None) -474,475: NewLine (None) -475,483: Indent (Indent(1)) -483,485: If (None) -486,494: Identifier (Str("response")) -494,495: . (None) -495,501: Identifier (Str("status")) -502,504: == (None) -505,508: Integer (Number("503")) -508,509: : (None) -509,510: NewLine (None) -510,522: Indent (Indent(1)) -522,528: Identifier (Str("logger")) -528,529: . (None) -529,533: Identifier (Str("info")) -533,534: ( (None) -534,536: FStringStart (Str("f\"")) -536,572: FstringMiddle (Str("Received 503 Service Unavailable on ")) -572,573: { (None) -573,581: Identifier (Str("date_obj")) -581,582: } (None) -582,595: FstringMiddle (Str(". Retrying...")) -595,596: FStringEnd (Str("\"")) -596,597: ) (None) -597,598: NewLine (None) -610,621: Identifier (Str("retry_count")) -622,624: += (None) -625,626: Integer (Number("1")) -626,627: NewLine (None) -639,644: Await (None) -645,652: Identifier (Str("asyncio")) -652,653: . (None) -653,658: Identifier (Str("sleep")) -658,659: ( (None) -659,660: Integer (Number("1")) -660,661: ) (None) -661,662: NewLine (None) -670,670: Dedent (Indent(1)) -670,674: Else (None) -674,675: : (None) -675,676: NewLine (None) -676,688: Indent (Indent(1)) -688,696: Identifier (Str("response")) -696,697: . (None) -697,713: Identifier (Str("raise_for_status")) -713,714: ( (None) -714,715: ) (None) -715,716: NewLine (None) -728,732: Identifier (Str("data")) -733,734: = (None) -735,740: Await (None) -741,749: Identifier (Str("response")) -749,750: . (None) -750,754: Identifier (Str("json")) -754,755: ( (None) -755,756: ) (None) -756,757: NewLine (None) -769,775: Identifier (Str("logger")) -775,776: . (None) -776,780: Identifier (Str("info")) -780,781: ( (None) -781,783: FStringStart (Str("f\"")) -783,821: FstringMiddle (Str("Successfully fetched trade details on ")) -821,822: { (None) -822,830: Identifier (Str("date_obj")) -830,831: } (None) -831,840: FstringMiddle (Str(" from tse")) -840,841: FStringEnd (Str("\"")) -841,842: ) (None) -842,843: NewLine (None) -855,861: Return (None) -862,863: [ (None) -863,871: Identifier (Str("date_obj")) -871,872: , (None) -873,875: Identifier (Str("pd")) -875,876: . (None) -876,890: Identifier (Str("json_normalize")) -890,891: ( (None) -891,895: Identifier (Str("data")) -895,896: [ (None) -896,910: StringLiteral (Str("\"tradeHistory\"")) -910,911: ] (None) -911,912: ) (None) -912,913: ] (None) -913,914: NewLine (None) -914,914: Dedent (Indent(1)) -914,914: Dedent (None) -914,914: Dedent (None) -914,920: Except (None) -921,922: ( (None) -922,929: Identifier (Str("aiohttp")) -929,930: . (None) -930,941: Identifier (Str("ClientError")) -941,942: , (None) -943,950: Identifier (Str("asyncio")) -950,951: . (None) -951,963: Identifier (Str("TimeoutError")) -963,964: ) (None) -964,965: : (None) -965,966: NewLine (None) -966,970: Indent (Indent(1)) -970,976: Identifier (Str("logger")) -976,977: . (None) -977,982: Identifier (Str("error")) -982,983: ( (None) -983,985: FStringStart (Str("f\"")) -985,1004: FstringMiddle (Str("Request failed for ")) -1004,1005: { (None) -1005,1013: Identifier (Str("date_obj")) -1013,1014: } (None) -1014,1027: FstringMiddle (Str(". Retrying...")) -1027,1028: FStringEnd (Str("\"")) -1028,1029: ) (None) -1029,1030: NewLine (None) -1034,1045: Identifier (Str("retry_count")) -1046,1048: += (None) -1049,1050: Integer (Number("1")) -1050,1051: NewLine (None) -1055,1060: Await (None) -1061,1068: Identifier (Str("asyncio")) -1068,1069: . (None) -1069,1074: Identifier (Str("sleep")) -1074,1075: ( (None) -1075,1076: Integer (Number("1")) -1076,1077: ) (None) -1077,1078: NewLine (None) -1078,1078: Dedent (Indent(1)) +0,52: Comment +52,53: NL +53,56: Try +56,57: : +57,58: NewLine +58,62: Indent +62,72: Identifier last_price +73,74: = +75,78: Identifier int +78,79: ( +79,92: Identifier price_section +92,93: [ +93,94: Integer 2 +94,95: ] +95,96: ) +96,97: NewLine +97,134: Comment +134,135: NL +135,135: Dedent +135,141: Except +142,143: ( +143,153: Identifier ValueError +153,154: , +155,165: Identifier IndexError +165,166: ) +166,167: : +167,168: NewLine +168,172: Indent +172,182: Identifier last_price +183,184: = +185,189: None +189,190: NewLine +190,190: Dedent +190,193: Try +193,194: : +194,195: NewLine +195,199: Indent +199,208: Identifier adj_close +209,210: = +211,214: Identifier int +214,215: ( +215,228: Identifier price_section +228,229: [ +229,230: Integer 3 +230,231: ] +231,232: ) +232,233: NewLine +233,233: Dedent +233,239: Except +240,241: ( +241,251: Identifier ValueError +251,252: , +253,263: Identifier IndexError +263,264: ) +264,265: : +265,266: NewLine +266,270: Indent +270,279: Identifier adj_close +280,281: = +282,286: None +286,287: NewLine +287,287: Dedent +287,290: Try +290,291: : +291,292: NewLine +292,296: Indent +296,306: Identifier market_cap +307,308: = +309,318: Identifier adj_close +319,320: * +321,325: Identifier self +325,326: . +326,338: Identifier total_shares +338,339: NewLine +339,339: Dedent +339,345: Except +346,356: Identifier ValueError +356,357: : +357,358: NewLine +358,362: Indent +362,372: Identifier market_cap +373,374: = +375,379: None +379,380: NewLine +380,381: NL +381,382: NL +382,382: Dedent +382,385: Try +385,386: : +386,387: NewLine +387,391: Indent +391,396: Async +397,401: With +402,409: Identifier session +409,410: . +410,413: Identifier get +413,414: ( +414,417: Identifier url +417,418: , +419,426: Identifier headers +426,427: = +427,447: Identifier TRADE_DETAILS_HEADER +447,448: , +449,456: Identifier timeout +456,457: = +457,460: Integer 100 +460,461: ) +462,464: As +465,473: Identifier response +473,474: : +474,475: NewLine +475,483: Indent +483,485: If +486,494: Identifier response +494,495: . +495,501: Identifier status +502,504: == +505,508: Integer 503 +508,509: : +509,510: NewLine +510,522: Indent +522,528: Identifier logger +528,529: . +529,533: Identifier info +533,534: ( +534,536: FStringStart +536,572: FstringMiddle Received 503 Service Unavailable on +572,573: { +573,581: Identifier date_obj +581,582: } +582,595: FstringMiddle . Retrying... +595,596: FStringEnd +596,597: ) +597,598: NewLine +610,621: Identifier retry_count +622,624: += +625,626: Integer 1 +626,627: NewLine +639,644: Await +645,652: Identifier asyncio +652,653: . +653,658: Identifier sleep +658,659: ( +659,660: Integer 1 +660,661: ) +661,662: NewLine +670,670: Dedent +670,674: Else +674,675: : +675,676: NewLine +676,688: Indent +688,696: Identifier response +696,697: . +697,713: Identifier raise_for_status +713,714: ( +714,715: ) +715,716: NewLine +728,732: Identifier data +733,734: = +735,740: Await +741,749: Identifier response +749,750: . +750,754: Identifier json +754,755: ( +755,756: ) +756,757: NewLine +769,775: Identifier logger +775,776: . +776,780: Identifier info +780,781: ( +781,783: FStringStart +783,821: FstringMiddle Successfully fetched trade details on +821,822: { +822,830: Identifier date_obj +830,831: } +831,840: FstringMiddle from tse +840,841: FStringEnd +841,842: ) +842,843: NewLine +855,861: Return +862,863: [ +863,871: Identifier date_obj +871,872: , +873,875: Identifier pd +875,876: . +876,890: Identifier json_normalize +890,891: ( +891,895: Identifier data +895,896: [ +896,910: StringLiteral "tradeHistory" +910,911: ] +911,912: ) +912,913: ] +913,914: NewLine +914,914: Dedent +914,914: Dedent +914,914: Dedent +914,920: Except +921,922: ( +922,929: Identifier aiohttp +929,930: . +930,941: Identifier ClientError +941,942: , +943,950: Identifier asyncio +950,951: . +951,963: Identifier TimeoutError +963,964: ) +964,965: : +965,966: NewLine +966,970: Indent +970,976: Identifier logger +976,977: . +977,982: Identifier error +982,983: ( +983,985: FStringStart +985,1004: FstringMiddle Request failed for +1004,1005: { +1005,1013: Identifier date_obj +1013,1014: } +1014,1027: FstringMiddle . Retrying... +1027,1028: FStringEnd +1028,1029: ) +1029,1030: NewLine +1034,1045: Identifier retry_count +1046,1048: += +1049,1050: Integer 1 +1050,1051: NewLine +1055,1060: Await +1061,1068: Identifier asyncio +1068,1069: . +1069,1074: Identifier sleep +1074,1075: ( +1075,1076: Integer 1 +1076,1077: ) +1077,1078: NewLine +1078,1078: Dedent diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@type_alias.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@type_alias.py.snap index 0d3ff891..7745af5b 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@type_alias.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@type_alias.py.snap @@ -3,24 +3,24 @@ source: parser/src/lexer/mod.rs description: "type Alias1 = int\ntype Alias2 = str\ntype Alias3 = float\n\ntype AliasToAnotherAlias = Alias1\n" input_file: parser/test_data/inputs/type_alias.py --- -0,4: Identifier (Type) -5,11: Identifier (Str("Alias1")) -12,13: = (None) -14,17: Identifier (Str("int")) -17,18: NewLine (None) -18,22: Identifier (Type) -23,29: Identifier (Str("Alias2")) -30,31: = (None) -32,35: Identifier (Str("str")) -35,36: NewLine (None) -36,40: Identifier (Type) -41,47: Identifier (Str("Alias3")) -48,49: = (None) -50,55: Identifier (Str("float")) -55,56: NewLine (None) -56,57: NL (None) -57,61: Identifier (Type) -62,81: Identifier (Str("AliasToAnotherAlias")) -82,83: = (None) -84,90: Identifier (Str("Alias1")) -90,91: NewLine (None) +0,4: type +5,11: Identifier Alias1 +12,13: = +14,17: Identifier int +17,18: NewLine +18,22: type +23,29: Identifier Alias2 +30,31: = +32,35: Identifier str +35,36: NewLine +36,40: type +41,47: Identifier Alias3 +48,49: = +50,55: Identifier float +55,56: NewLine +56,57: NL +57,61: type +62,81: Identifier AliasToAnotherAlias +82,83: = +84,90: Identifier Alias1 +90,91: NewLine diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@with.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@with.py.snap index 1f875ec6..05c5b600 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@with.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@with.py.snap @@ -3,89 +3,89 @@ source: parser/src/lexer/mod.rs description: "with a: pass\n\nwith a as b: pass\n\nwith a as b, c as d: pass\n\nwith (a as b, c as d): pass\n\nasync with a as b: pass\n\nasync with a as b, c as d:\n pass\n\nasync with (\n a as b, c as d\n):\n a = 1\n" input_file: parser/test_data/inputs/with.py --- -0,4: With (None) -5,6: Identifier (Str("a")) -6,7: : (None) -8,12: Pass (None) -12,13: NewLine (None) -13,14: NL (None) -14,18: With (None) -19,20: Identifier (Str("a")) -21,23: As (None) -24,25: Identifier (Str("b")) -25,26: : (None) -27,31: Pass (None) -31,32: NewLine (None) -32,33: NL (None) -33,37: With (None) -38,39: Identifier (Str("a")) -40,42: As (None) -43,44: Identifier (Str("b")) -44,45: , (None) -46,47: Identifier (Str("c")) -48,50: As (None) -51,52: Identifier (Str("d")) -52,53: : (None) -54,58: Pass (None) -58,59: NewLine (None) -59,60: NL (None) -60,64: With (None) -65,66: ( (None) -66,67: Identifier (Str("a")) -68,70: As (None) -71,72: Identifier (Str("b")) -72,73: , (None) -74,75: Identifier (Str("c")) -76,78: As (None) -79,80: Identifier (Str("d")) -80,81: ) (None) -81,82: : (None) -83,87: Pass (None) -87,88: NewLine (None) -88,89: NL (None) -89,94: Async (None) -95,99: With (None) -100,101: Identifier (Str("a")) -102,104: As (None) -105,106: Identifier (Str("b")) -106,107: : (None) -108,112: Pass (None) -112,113: NewLine (None) -113,114: NL (None) -114,119: Async (None) -120,124: With (None) -125,126: Identifier (Str("a")) -127,129: As (None) -130,131: Identifier (Str("b")) -131,132: , (None) -133,134: Identifier (Str("c")) -135,137: As (None) -138,139: Identifier (Str("d")) -139,140: : (None) -140,141: NewLine (None) -141,145: Indent (Indent(1)) -145,149: Pass (None) -149,150: NewLine (None) -150,151: NL (None) -151,151: Dedent (Indent(1)) -151,156: Async (None) -157,161: With (None) -162,163: ( (None) -163,164: NL (None) -172,173: Identifier (Str("a")) -174,176: As (None) -177,178: Identifier (Str("b")) -178,179: , (None) -180,181: Identifier (Str("c")) -182,184: As (None) -185,186: Identifier (Str("d")) -186,187: NL (None) -187,188: ) (None) -188,189: : (None) -189,190: NewLine (None) -190,194: Indent (Indent(1)) -194,195: Identifier (Str("a")) -196,197: = (None) -198,199: Integer (Number("1")) -199,200: NewLine (None) -200,200: Dedent (Indent(1)) +0,4: With +5,6: Identifier a +6,7: : +8,12: Pass +12,13: NewLine +13,14: NL +14,18: With +19,20: Identifier a +21,23: As +24,25: Identifier b +25,26: : +27,31: Pass +31,32: NewLine +32,33: NL +33,37: With +38,39: Identifier a +40,42: As +43,44: Identifier b +44,45: , +46,47: Identifier c +48,50: As +51,52: Identifier d +52,53: : +54,58: Pass +58,59: NewLine +59,60: NL +60,64: With +65,66: ( +66,67: Identifier a +68,70: As +71,72: Identifier b +72,73: , +74,75: Identifier c +76,78: As +79,80: Identifier d +80,81: ) +81,82: : +83,87: Pass +87,88: NewLine +88,89: NL +89,94: Async +95,99: With +100,101: Identifier a +102,104: As +105,106: Identifier b +106,107: : +108,112: Pass +112,113: NewLine +113,114: NL +114,119: Async +120,124: With +125,126: Identifier a +127,129: As +130,131: Identifier b +131,132: , +133,134: Identifier c +135,137: As +138,139: Identifier d +139,140: : +140,141: NewLine +141,145: Indent +145,149: Pass +149,150: NewLine +150,151: NL +151,151: Dedent +151,156: Async +157,161: With +162,163: ( +163,164: NL +172,173: Identifier a +174,176: As +177,178: Identifier b +178,179: , +180,181: Identifier c +182,184: As +185,186: Identifier d +186,187: NL +187,188: ) +188,189: : +189,190: NewLine +190,194: Indent +194,195: Identifier a +196,197: = +198,199: Integer 1 +199,200: NewLine +200,200: Dedent diff --git a/typechecker/src/file.rs b/typechecker/src/file.rs index 6f0779d1..6b251b81 100755 --- a/typechecker/src/file.rs +++ b/typechecker/src/file.rs @@ -1,4 +1,5 @@ use core::panic; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::path::PathBuf; use std::sync::atomic::AtomicUsize; use std::sync::Arc; @@ -58,8 +59,18 @@ impl<'a> EnderpyFile { let source = std::fs::read_to_string(&path).unwrap_or_else(|_| panic!("cannot read file {path:?}")); let module = get_module_name(&path); - let mut parser = Parser::new(&source, ""); - let tree = parser.parse().expect("parsing {path:?} failed"); + + let mut parser = Parser::new(&source); + let parse_result = catch_unwind(AssertUnwindSafe(|| parser.parse())); + let tree = match parse_result { + Ok(Ok(module)) => module, + Ok(Err(_)) => { + panic!("Cannot parse file : {path:?}"); + } + Err(_) => { + panic!("Cannot parse file : {path:?}"); + } + }; let line_starts = parser.lexer.line_starts.clone(); let id = if path.ends_with("builtins.pyi") { diff --git a/typechecker/src/type_evaluator.rs b/typechecker/src/type_evaluator.rs index c62a90a1..11d76cda 100755 --- a/typechecker/src/type_evaluator.rs +++ b/typechecker/src/type_evaluator.rs @@ -562,7 +562,7 @@ impl<'a> TypeEvaluator<'a> { // then local scope. // https://peps.python.org/pep-0563/#backwards-compatibility ast::ConstantValue::Str(ref str) => { - let mut parser = Parser::new(str, ""); + let mut parser = Parser::new(str); // Wrap the parsing logic inside a `catch_unwind` block let parse_result = catch_unwind(AssertUnwindSafe(|| parser.parse()));