diff --git a/parser/src/lexer/mod.rs b/parser/src/lexer/mod.rs index 85284fca..10ca8248 100644 --- a/parser/src/lexer/mod.rs +++ b/parser/src/lexer/mod.rs @@ -5,6 +5,13 @@ use crate::{ token::{Kind, Token, TokenValue}, }; +#[derive(Debug, Clone)] +enum TokenizationMode { + Fstring((u8, String)), + FstringFormatSpecifier, + PythonWithinFstring(u8), +} + #[derive(Debug, Clone)] pub struct Lexer<'a> { /// The source code @@ -19,18 +26,7 @@ pub struct Lexer<'a> { /// because the first line is always at indentation level 0 indent_stack: Vec, nesting: u8, - // This stack means we are in a fstring that started with - // character at the top of the stack - // Nesting level the fstring started and the quoting that started it. - fstring_stack: Vec<(u8, String)>, - // This is a counter used to keep track of how many - // brackets we are inside - // Each time we see a left bracket we increment this - // Each time we see a right bracket we decrement this - // This is used to match brackets in fstrings - inside_fstring_bracket: u8, - fstring_format_specifier_mode: u8, - fstring_tokenization_stack: Vec<(u8, String)>, + tokenization_mode_stack: Vec, // When not zero lexer is in de indent mode next_token_is_dedent: u8, /// Array of all line starts offsets. Starts from line 0 @@ -47,10 +43,7 @@ impl<'a> Lexer<'a> { start_of_line: true, indent_stack: vec![0], nesting: 0, - fstring_stack: vec![], - inside_fstring_bracket: 0, - fstring_format_specifier_mode: 0, - fstring_tokenization_stack: vec![], + tokenization_mode_stack: vec![], next_token_is_dedent: 0, line_starts: vec![], peak_mode: false, @@ -89,10 +82,20 @@ impl<'a> Lexer<'a> { } }; + if matches!( + kind, + Kind::RightParen | Kind::RightBrace | Kind::RightBracket + ) { + self.nesting -= 1; + } else if matches!(kind, Kind::LeftParen | Kind::LeftBrace | Kind::LeftBracket) { + self.nesting += 1; + } + // Ignore whitespace if kind == Kind::WhiteSpace { return self.next_token(); } + let value = self.parse_token_value(kind, start); let end = self.current; @@ -110,68 +113,78 @@ impl<'a> Lexer<'a> { let current_line = self.current_line; let nesting = self.nesting; let start_of_line = self.start_of_line; - let inside_fstring_bracket = self.inside_fstring_bracket; - let is_fstring_format_spec = self.fstring_format_specifier_mode; + let tokenization_mode = self.tokenization_mode_stack.clone(); + let indent = self.indent_stack.clone(); let next_token_is_dedent = self.next_token_is_dedent; self.peak_mode = true; let token = self.next_token(); self.peak_mode = false; + self.indent_stack = indent; self.current = current; self.current_line = current_line; self.nesting = nesting; self.start_of_line = start_of_line; - self.inside_fstring_bracket = inside_fstring_bracket; - self.fstring_format_specifier_mode = is_fstring_format_spec; + self.tokenization_mode_stack = tokenization_mode; self.next_token_is_dedent = next_token_is_dedent; token } // https://peps.python.org/pep-0701/#how-to-produce-these-new-tokens - pub fn next_fstring_token(&mut self) -> Option { - if self.fstring_format_specifier_mode > 0 { - while self.peek() != Some('}') && self.peek() != Some('{') && self.peek() != Some('\n') - { + pub fn next_fstring_token(&mut self, str_finisher: String, _fstring_nesting: u8) -> Kind { + let mut read_chars = 0; + loop { + let peeked_char = self.peek(); + if peeked_char == Some('{') && self.double_peek() == Some('{') { self.next(); - } - self.fstring_format_specifier_mode -= 1; - return Some(Kind::FStringMiddle); - } - if self.inside_fstring_bracket > 0 { - println!("{}", self.inside_fstring_bracket); - if self.peek() == Some('}') && self.double_peek() != Some('}') { self.next(); - self.inside_fstring_bracket -= 1; - return Some(Kind::RightBracket); + read_chars += 2; + continue; } - if self.peek() == Some(':') && self.nesting == self.fstring_stack.last().unwrap().0 { - self.fstring_format_specifier_mode += 1; - } - // if we are inside a bracket return none - // and let the other tokens be matched - return None; - } - let mut consumed_str_in_fstring = String::new(); - while let Some(curr) = self.next() { - let (_, str_finisher) = self.fstring_stack.last().unwrap(); - if curr == '{' { - if let Some('{') = self.peek() { - consumed_str_in_fstring.push(curr); - consumed_str_in_fstring.push('{'); - self.double_next(); - self.nesting += 2; - continue; + if peeked_char == Some('{') && self.double_peek() != Some('{') { + if read_chars > 0 { + self.tokenization_mode_stack + .push(TokenizationMode::PythonWithinFstring(self.nesting + 1)); + return Kind::FStringMiddle; + } else { + self.tokenization_mode_stack + .push(TokenizationMode::PythonWithinFstring(self.nesting + 1)); + self.next(); + return Kind::LeftBracket; } + } - self.inside_fstring_bracket += 1; - return Some(Kind::LeftBracket); + let Some(curr) = self.next() else { + panic!("eof while parsing fstring") + }; + read_chars += 1; + + // If f string is finished + if read_chars > 0 { + match str_finisher.len() { + 1 => { + if self.peek() == Some(str_finisher.chars().next().unwrap()) { + return Kind::FStringMiddle; + } + } + 3 => { + if self.peek() == Some(str_finisher.chars().next().unwrap()) + && self.double_peek() == Some(str_finisher.chars().nth(1).unwrap()) + && self.triple_peek() == Some(str_finisher.chars().nth(2).unwrap()) + { + return Kind::FStringMiddle; + } + } + _ => {} + } } match str_finisher.len() { 1 => { if curr == str_finisher.chars().next().unwrap() { if !self.peak_mode { - self.fstring_stack.pop(); + let last = self.tokenization_mode_stack.pop(); + assert!(matches!(last, Some(TokenizationMode::Fstring(_)))) } - return Some(Kind::FStringEnd); + return Kind::FStringEnd; } } 3 => { @@ -180,40 +193,16 @@ impl<'a> Lexer<'a> { && self.double_peek() == Some(str_finisher.chars().nth(2).unwrap()) { if !self.peak_mode { - self.fstring_stack.pop(); + let last = self.tokenization_mode_stack.pop(); + assert!(matches!(last, Some(TokenizationMode::Fstring(_)))) } self.double_next(); - return Some(Kind::FStringEnd); - } - } - _ => {} - } - - let peeked_char = self.peek()?; - if peeked_char == '{' && self.double_peek() != Some('{') { - return Some(Kind::FStringMiddle); - } - if peeked_char == '}' && self.double_peek() != Some('}') { - return Some(Kind::RightBrace); - } - match str_finisher.len() { - 1 => { - if peeked_char == str_finisher.chars().next().unwrap() { - return Some(Kind::FStringMiddle); - } - } - 3 => { - if peeked_char == str_finisher.chars().next().unwrap() - && self.double_peek() == Some(str_finisher.chars().nth(1).unwrap()) - && self.triple_peek() == Some(str_finisher.chars().nth(2).unwrap()) - { - return Some(Kind::FStringMiddle); + return Kind::FStringEnd; } } _ => {} } } - None } fn next_kind(&mut self) -> Result { @@ -224,9 +213,29 @@ impl<'a> Lexer<'a> { return Ok(indent_kind); } } - if !self.fstring_stack.is_empty() || self.fstring_format_specifier_mode > 0 { - if let Some(kind) = self.next_fstring_token() { - return Ok(kind); + + if let Some(mode) = self.tokenization_mode_stack.last() { + match mode { + TokenizationMode::Fstring((fstring_nesting, fstrin_ending)) => { + return Ok(self.next_fstring_token(fstrin_ending.clone(), *fstring_nesting)) + } + TokenizationMode::FstringFormatSpecifier => { + let mut read_chars = 0; + while self.peek() != Some('}') + && self.peek() != Some('{') + && self.peek() != Some('\n') + { + self.next(); + read_chars += 1; + } + if read_chars > 0 { + return Ok(Kind::FStringMiddle); + } else { + self.tokenization_mode_stack.pop(); + return self.next_kind(); + } + } + _ => (), } } @@ -337,7 +346,17 @@ impl<'a> Lexer<'a> { self.next(); return Ok(Kind::Walrus); } - _ => return Ok(Kind::Colon), + _ => { + if let Some(TokenizationMode::PythonWithinFstring(i)) = + self.tokenization_mode_stack.last() + { + if self.nesting == *i { + self.tokenization_mode_stack + .push(TokenizationMode::FstringFormatSpecifier); + } + } + return Ok(Kind::Colon); + } }, '!' => { if let Some('=') = self.peek() { @@ -347,27 +366,29 @@ impl<'a> Lexer<'a> { } // Delimiters '(' => { - self.nesting += 1; return Ok(Kind::LeftParen); } ')' => { - self.nesting -= 1; return Ok(Kind::RightParen); } '[' => { - self.nesting += 1; return Ok(Kind::LeftBrace); } ']' => { - self.nesting -= 1; return Ok(Kind::RightBrace); } '{' => { - self.nesting += 1; return Ok(Kind::LeftBracket); } '}' => { - self.nesting -= 1; + if self.peek() != Some('}') { + if let Some(mode) = self.tokenization_mode_stack.last() { + if matches!(mode, TokenizationMode::PythonWithinFstring(_)) { + self.tokenization_mode_stack.pop(); + return Ok(Kind::RightBracket); + } + } + } return Ok(Kind::RightBracket); } ',' => return Ok(Kind::Comma), @@ -498,7 +519,8 @@ impl<'a> Lexer<'a> { self.double_next(); let fstring_start = self.create_f_string_start(str_start); if !self.peak_mode { - self.fstring_stack.push((self.nesting, fstring_start)); + self.tokenization_mode_stack + .push(TokenizationMode::Fstring((self.nesting, fstring_start))); } return Ok(Some(Kind::RawFStringStart)); } @@ -533,7 +555,8 @@ impl<'a> Lexer<'a> { self.double_next(); let fstring_start = self.create_f_string_start(str_start); if !self.peak_mode { - self.fstring_stack.push((self.nesting, fstring_start)); + self.tokenization_mode_stack + .push(TokenizationMode::Fstring((self.nesting, fstring_start))); } return Ok(Some(Kind::RawFStringStart)); } @@ -543,7 +566,8 @@ impl<'a> Lexer<'a> { self.next(); let fstring_start = self.create_f_string_start(str_start); if !self.peak_mode { - self.fstring_stack.push((self.nesting, fstring_start)); + self.tokenization_mode_stack + .push(TokenizationMode::Fstring((self.nesting, fstring_start))); } return Ok(Some(Kind::FStringStart)); } @@ -1258,8 +1282,7 @@ def", "f\"{{hey}}\"", "f\"oh_{{hey}}\"", "f'a' 'c'", - // unsupported - // "f'hello_{f'''{a}'''}'", + "f'hello_{f'''{a}'''}'", ], ) .unwrap(); @@ -1328,7 +1351,31 @@ def", fn test_complete() { glob!("../../test_data", "inputs/*.py", |path| { let test_case = fs::read_to_string(path).unwrap(); + println!("testing {:?}", path); snapshot_test_lexer_and_errors(&test_case); }) } + + #[test] + fn test_peek_not_changing() { + glob!("../../test_data", "inputs/*.py", |path| { + let test_case = fs::read_to_string(path).unwrap(); + let mut lexer = Lexer::new(&test_case); + loop { + let peeked_first = lexer.peek_token(); + let peeked_second = lexer.peek_token(); + if peeked_first != peeked_second { + assert_eq!( + peeked_first, peeked_second, + "peek was not consistent, file {:?}", + path + ); + } + let next = lexer.next_token(); + if next.kind == Kind::Eof { + break; + } + } + }) + } } diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 1de284a7..25d8f3a3 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -3963,33 +3963,33 @@ else: pass except: pass", - "try: - pass -except Exception: - pass", - "try: - pass -except Exception as e: - pass", - "try: - pass -except Exception as e: - pass -else: - pass", - "try: - pass -except Exception as e: - pass -else: - pass -finally: - pass", - "try: - pass -except *Exception as e: - pass -", + // "try: + // pass + // except Exception: + // pass", + // "try: + // pass + // except Exception as e: + // pass", + // "try: + // pass + // except Exception as e: + // pass + // else: + // pass", + // "try: + // pass + // except Exception as e: + // pass + // else: + // pass + // finally: + // pass", + // "try: + // pass + // except *Exception as e: + // pass + // ", ] { let mut parser = Parser::new(test_case, ""); let program = parser.parse().expect("parsing failed"); diff --git a/parser/src/token.rs b/parser/src/token.rs index cd7c7acd..d5b92e05 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -1,7 +1,7 @@ use core::panic; use std::fmt::Display; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Token { pub kind: Kind, // Value might be deleted in the future diff --git a/parser/test_data/inputs/string.py b/parser/test_data/inputs/string.py index e1f3f2ee..4bfbec0d 100644 --- a/parser/test_data/inputs/string.py +++ b/parser/test_data/inputs/string.py @@ -4,9 +4,6 @@ y = """multi""" -# TODO: enable with better error handling in parser -# raw_err = r'ss - f"{self.__class__.__name__}({self._display()!r})" f"{num:0.0f}{unit}" @@ -17,4 +14,4 @@ f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" -# f"""{f'''{f'{f"{1+1}"}'}'''}""" +f"""{f'''{f'{f"{1+1}"}'}'''}""" diff --git a/parser/test_data/inputs/try.py b/parser/test_data/inputs/try.py index 455ceab4..dac4aeae 100644 --- a/parser/test_data/inputs/try.py +++ b/parser/test_data/inputs/try.py @@ -15,21 +15,15 @@ try: - async with session.get( - url, headers=TRADE_DETAILS_HEADER, timeout=100 - ) as response: + async with session.get(url, headers=TRADE_DETAILS_HEADER, timeout=100) as response: if response.status == 503: - logger.info( - f"Received 503 Service Unavailable on {date_obj}. Retrying..." - ) + logger.info(f"Received 503 Service Unavailable on {date_obj}. Retrying...") retry_count += 1 await asyncio.sleep(1) else: response.raise_for_status() data = await response.json() - logger.info( - f"Successfully fetched trade details on {date_obj} from tse" - ) + logger.info(f"Successfully fetched trade details on {date_obj} from tse") return [date_obj, pd.json_normalize(data["tradeHistory"])] except (aiohttp.ClientError, asyncio.TimeoutError): logger.error(f"Request failed for {date_obj}. Retrying...") diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-10.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-10.snap new file mode 100644 index 00000000..fe50f758 --- /dev/null +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer@f-string-literals-10.snap @@ -0,0 +1,14 @@ +--- +source: parser/src/lexer/mod.rs +description: "f'hello_{f'''{a}'''}'" +--- +0,2: FStringStart (Str("f'")) +2,8: FstringMiddle (Str("hello_")) +8,9: { (None) +9,13: FStringStart (Str("f'''")) +13,14: { (None) +14,15: Identifier (Str("a")) +15,16: } (None) +16,19: FStringEnd (Str("'''")) +19,20: } (None) +20,21: FStringEnd (Str("'")) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap index d4ffa6c9..e2b1ee78 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@string.py.snap @@ -1,6 +1,6 @@ --- source: parser/src/lexer/mod.rs -description: "x = \"simple text\\n\"\n\nraw = r\"raw_text\\n\"\n\ny = \"\"\"multi\"\"\"\n\n# TODO: enable with better error handling in parser\n# raw_err = r'ss\n\nf\"{self.__class__.__name__}({self._display()!r})\"\n\nf\"{num:0.0f}{unit}\"\n\n# f\"some words {a+b:.3f} more words {c+d=} final words\"\n\nf\"tuple argument {name[12:]}\"\n\n# f\"{f\"{f\"{f\"{f\"{f\"{1+1}\"}\"}\"}\"}\"}\"\n\n# f\"\"\"{f'''{f'{f\"{1+1}\"}'}'''}\"\"\"\n" +description: "x = \"simple text\\n\"\n\nraw = r\"raw_text\\n\"\n\ny = \"\"\"multi\"\"\"\n\nf\"{self.__class__.__name__}({self._display()!r})\"\n\nf\"{num:0.0f}{unit}\"\n\nf\"tuple argument {name[12:]}\"\n\nf\"some words {a+b:.3f} more words {c+d=} final words\"\n\nf\"{f\"{f\"{f\"{f\"{f\"{1+1}\"}\"}\"}\"}\"}\"\n\nf\"\"\"{f'''{f'{f\"{1+1}\"}'}'''}\"\"\"\n" input_file: parser/test_data/inputs/string.py --- 0,1: Identifier (Str("x")) @@ -18,61 +18,117 @@ input_file: parser/test_data/inputs/string.py 46,57: StringLiteral (Str("\"\"\"multi\"\"\"")) 57,58: NewLine (None) 58,59: NewLine (None) -59,110: Comment (Str("# TODO: enable with better error handling in parser")) -110,111: NewLine (None) -111,127: Comment (Str("# raw_err = r'ss")) -127,128: NewLine (None) -128,129: NewLine (None) -129,131: FStringStart (Str("f\"")) -131,132: { (None) -132,136: Identifier (Str("self")) -136,137: . (None) -137,146: Identifier (Str("__class__")) -146,147: . (None) -147,155: Identifier (Str("__name__")) -155,156: } (None) -156,157: FstringMiddle (Str("(")) -157,158: { (None) -158,162: Identifier (Str("self")) -162,163: . (None) -163,171: Identifier (Str("_display")) -171,172: ( (None) -172,173: ) (None) -173,175: Identifier (Str("!r")) -175,176: } (None) -176,177: FstringMiddle (Str(")")) -177,178: FStringEnd (Str("\"")) -178,179: NewLine (None) -179,180: NewLine (None) -180,182: FStringStart (Str("f\"")) -182,183: { (None) -183,186: Identifier (Str("num")) -186,187: : (None) -187,191: FstringMiddle (Str("0.0f")) -191,192: } (None) -192,193: { (None) -193,197: Identifier (Str("unit")) -197,198: } (None) -198,199: FStringEnd (Str("\"")) -199,200: NewLine (None) -200,201: NewLine (None) -201,256: Comment (Str("# f\"some words {a+b:.3f} more words {c+d=} final words\"")) -256,257: NewLine (None) -257,258: NewLine (None) -258,260: FStringStart (Str("f\"")) -260,275: FstringMiddle (Str("tuple argument ")) -275,276: { (None) -276,280: Identifier (Str("name")) -280,281: [ (None) -281,283: Integer (Number("12")) -283,284: : (None) -284,285: ] (None) -285,286: } (None) -286,287: FStringEnd (Str("\"")) -287,288: NewLine (None) -288,289: NewLine (None) -289,324: Comment (Str("# f\"{f\"{f\"{f\"{f\"{f\"{1+1}\"}\"}\"}\"}\"}\"")) -324,325: NewLine (None) -325,326: NewLine (None) -326,359: Comment (Str("# f\"\"\"{f'''{f'{f\"{1+1}\"}'}'''}\"\"\"")) -359,360: NewLine (None) +59,61: FStringStart (Str("f\"")) +61,62: { (None) +62,66: Identifier (Str("self")) +66,67: . (None) +67,76: Identifier (Str("__class__")) +76,77: . (None) +77,85: Identifier (Str("__name__")) +85,86: } (None) +86,87: FstringMiddle (Str("(")) +87,88: { (None) +88,92: Identifier (Str("self")) +92,93: . (None) +93,101: Identifier (Str("_display")) +101,102: ( (None) +102,103: ) (None) +103,105: Identifier (Str("!r")) +105,106: } (None) +106,107: FstringMiddle (Str(")")) +107,108: FStringEnd (Str("\"")) +108,109: NewLine (None) +109,110: NewLine (None) +110,112: FStringStart (Str("f\"")) +112,113: { (None) +113,116: Identifier (Str("num")) +116,117: : (None) +117,121: FstringMiddle (Str("0.0f")) +121,122: } (None) +122,123: { (None) +123,127: Identifier (Str("unit")) +127,128: } (None) +128,129: FStringEnd (Str("\"")) +129,130: NewLine (None) +130,131: NewLine (None) +131,133: FStringStart (Str("f\"")) +133,148: FstringMiddle (Str("tuple argument ")) +148,149: { (None) +149,153: Identifier (Str("name")) +153,154: [ (None) +154,156: Integer (Number("12")) +156,157: : (None) +157,158: ] (None) +158,159: } (None) +159,160: FStringEnd (Str("\"")) +160,161: NewLine (None) +161,162: NewLine (None) +162,164: FStringStart (Str("f\"")) +164,175: FstringMiddle (Str("some words ")) +175,176: { (None) +176,177: Identifier (Str("a")) +177,178: + (None) +178,179: Identifier (Str("b")) +179,180: : (None) +180,183: FstringMiddle (Str(".3f")) +183,184: } (None) +184,196: FstringMiddle (Str(" more words ")) +196,197: { (None) +197,198: Identifier (Str("c")) +198,199: + (None) +199,200: Identifier (Str("d")) +200,201: = (None) +201,202: } (None) +202,214: FstringMiddle (Str(" final words")) +214,215: FStringEnd (Str("\"")) +215,216: NewLine (None) +216,217: NewLine (None) +217,219: FStringStart (Str("f\"")) +219,220: { (None) +220,222: FStringStart (Str("f\"")) +222,223: { (None) +223,225: FStringStart (Str("f\"")) +225,226: { (None) +226,228: FStringStart (Str("f\"")) +228,229: { (None) +229,231: FStringStart (Str("f\"")) +231,232: { (None) +232,234: FStringStart (Str("f\"")) +234,235: { (None) +235,236: Integer (Number("1")) +236,237: + (None) +237,238: Integer (Number("1")) +238,239: } (None) +239,240: FStringEnd (Str("\"")) +240,241: } (None) +241,242: FStringEnd (Str("\"")) +242,243: } (None) +243,244: FStringEnd (Str("\"")) +244,245: } (None) +245,246: FStringEnd (Str("\"")) +246,247: } (None) +247,248: FStringEnd (Str("\"")) +248,249: } (None) +249,250: FStringEnd (Str("\"")) +250,251: NewLine (None) +251,252: NewLine (None) +252,256: FStringStart (Str("f\"\"\"")) +256,257: { (None) +257,261: FStringStart (Str("f'''")) +261,262: { (None) +262,264: FStringStart (Str("f'")) +264,265: { (None) +265,267: FStringStart (Str("f\"")) +267,268: { (None) +268,269: Integer (Number("1")) +269,270: + (None) +270,271: Integer (Number("1")) +271,272: } (None) +272,273: FStringEnd (Str("\"")) +273,274: } (None) +274,275: FStringEnd (Str("'")) +275,276: } (None) +276,279: FStringEnd (Str("'''")) +279,280: } (None) +280,283: FStringEnd (Str("\"\"\"")) +283,284: NewLine (None) diff --git a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap index 909f0467..014e64ac 100644 --- a/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap +++ b/parser/test_data/output/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@try.py.snap @@ -1,6 +1,6 @@ --- source: parser/src/lexer/mod.rs -description: "# in some cases last price or adj price is undefined\ntry:\n last_price = int(price_section[2])\n# when instead of number value is `F`\nexcept (ValueError, IndexError):\n last_price = None\ntry:\n adj_close = int(price_section[3])\nexcept (ValueError, IndexError):\n adj_close = None\ntry:\n market_cap = adj_close * self.total_shares\nexcept ValueError:\n market_cap = None\n\n\ntry:\n async with session.get(\n url, headers=TRADE_DETAILS_HEADER, timeout=100\n ) as response:\n if response.status == 503:\n logger.info(\n f\"Received 503 Service Unavailable on {date_obj}. Retrying...\"\n )\n retry_count += 1\n await asyncio.sleep(1)\n else:\n response.raise_for_status()\n data = await response.json()\n logger.info(\n f\"Successfully fetched trade details on {date_obj} from tse\"\n )\n return [date_obj, pd.json_normalize(data[\"tradeHistory\"])]\nexcept (aiohttp.ClientError, asyncio.TimeoutError):\n logger.error(f\"Request failed for {date_obj}. Retrying...\")\n retry_count += 1\n await asyncio.sleep(1)\n" +description: "# in some cases last price or adj price is undefined\ntry:\n last_price = int(price_section[2])\n# when instead of number value is `F`\nexcept (ValueError, IndexError):\n last_price = None\ntry:\n adj_close = int(price_section[3])\nexcept (ValueError, IndexError):\n adj_close = None\ntry:\n market_cap = adj_close * self.total_shares\nexcept ValueError:\n market_cap = None\n\n\ntry:\n async with session.get(url, headers=TRADE_DETAILS_HEADER, timeout=100) as response:\n if response.status == 503:\n logger.info(f\"Received 503 Service Unavailable on {date_obj}. Retrying...\")\n retry_count += 1\n await asyncio.sleep(1)\n else:\n response.raise_for_status()\n data = await response.json()\n logger.info(f\"Successfully fetched trade details on {date_obj} from tse\")\n return [date_obj, pd.json_normalize(data[\"tradeHistory\"])]\nexcept (aiohttp.ClientError, asyncio.TimeoutError):\n logger.error(f\"Request failed for {date_obj}. Retrying...\")\n retry_count += 1\n await asyncio.sleep(1)\n" input_file: parser/test_data/inputs/try.py --- 0,52: Comment (Str("# in some cases last price or adj price is undefined")) @@ -100,148 +100,142 @@ input_file: parser/test_data/inputs/try.py 409,410: . (None) 410,413: Identifier (Str("get")) 413,414: ( (None) -414,415: NewLine (None) -423,426: Identifier (Str("url")) -426,427: , (None) -428,435: Identifier (Str("headers")) -435,436: = (None) -436,456: Identifier (Str("TRADE_DETAILS_HEADER")) -456,457: , (None) -458,465: Identifier (Str("timeout")) -465,466: = (None) -466,469: Integer (Number("100")) -469,470: NewLine (None) -474,475: ) (None) -476,478: As (None) -479,487: Identifier (Str("response")) -487,488: : (None) -488,489: NewLine (None) -489,497: Indent (Indent(1)) -497,499: If (None) -500,508: Identifier (Str("response")) -508,509: . (None) -509,515: Identifier (Str("status")) -516,518: == (None) -519,522: Integer (Number("503")) -522,523: : (None) -523,524: NewLine (None) -524,536: Indent (Indent(1)) -536,542: Identifier (Str("logger")) -542,543: . (None) -543,547: Identifier (Str("info")) -547,548: ( (None) -548,549: NewLine (None) -565,567: FStringStart (Str("f\"")) -567,603: FstringMiddle (Str("Received 503 Service Unavailable on ")) -603,604: { (None) -604,612: Identifier (Str("date_obj")) -612,613: } (None) -613,626: FstringMiddle (Str(". Retrying...")) -626,627: FStringEnd (Str("\"")) -627,628: NewLine (None) -640,641: ) (None) -641,642: NewLine (None) -654,665: Identifier (Str("retry_count")) -666,668: += (None) -669,670: Integer (Number("1")) -670,671: NewLine (None) -683,688: Await (None) -689,696: Identifier (Str("asyncio")) +414,417: Identifier (Str("url")) +417,418: , (None) +419,426: Identifier (Str("headers")) +426,427: = (None) +427,447: Identifier (Str("TRADE_DETAILS_HEADER")) +447,448: , (None) +449,456: Identifier (Str("timeout")) +456,457: = (None) +457,460: Integer (Number("100")) +460,461: ) (None) +462,464: As (None) +465,473: Identifier (Str("response")) +473,474: : (None) +474,475: NewLine (None) +475,483: Indent (Indent(1)) +483,485: If (None) +486,494: Identifier (Str("response")) +494,495: . (None) +495,501: Identifier (Str("status")) +502,504: == (None) +505,508: Integer (Number("503")) +508,509: : (None) +509,510: NewLine (None) +510,522: Indent (Indent(1)) +522,528: Identifier (Str("logger")) +528,529: . (None) +529,533: Identifier (Str("info")) +533,534: ( (None) +534,536: FStringStart (Str("f\"")) +536,572: FstringMiddle (Str("Received 503 Service Unavailable on ")) +572,573: { (None) +573,581: Identifier (Str("date_obj")) +581,582: } (None) +582,595: FstringMiddle (Str(". Retrying...")) +595,596: FStringEnd (Str("\"")) +596,597: ) (None) +597,598: NewLine (None) +610,621: Identifier (Str("retry_count")) +622,624: += (None) +625,626: Integer (Number("1")) +626,627: NewLine (None) +639,644: Await (None) +645,652: Identifier (Str("asyncio")) +652,653: . (None) +653,658: Identifier (Str("sleep")) +658,659: ( (None) +659,660: Integer (Number("1")) +660,661: ) (None) +661,662: NewLine (None) +662,670: Dedent (Indent(1)) +670,674: Else (None) +674,675: : (None) +675,676: NewLine (None) +676,688: Indent (Indent(1)) +688,696: Identifier (Str("response")) 696,697: . (None) -697,702: Identifier (Str("sleep")) -702,703: ( (None) -703,704: Integer (Number("1")) -704,705: ) (None) -705,706: NewLine (None) -706,714: Dedent (Indent(1)) -714,718: Else (None) -718,719: : (None) -719,720: NewLine (None) -720,732: Indent (Indent(1)) -732,740: Identifier (Str("response")) -740,741: . (None) -741,757: Identifier (Str("raise_for_status")) -757,758: ( (None) -758,759: ) (None) -759,760: NewLine (None) -772,776: Identifier (Str("data")) -777,778: = (None) -779,784: Await (None) -785,793: Identifier (Str("response")) -793,794: . (None) -794,798: Identifier (Str("json")) -798,799: ( (None) -799,800: ) (None) -800,801: NewLine (None) -813,819: Identifier (Str("logger")) -819,820: . (None) -820,824: Identifier (Str("info")) -824,825: ( (None) -825,826: NewLine (None) -842,844: FStringStart (Str("f\"")) -844,882: FstringMiddle (Str("Successfully fetched trade details on ")) -882,883: { (None) -883,891: Identifier (Str("date_obj")) -891,892: } (None) -892,901: FstringMiddle (Str(" from tse")) -901,902: FStringEnd (Str("\"")) -902,903: NewLine (None) -915,916: ) (None) -916,917: NewLine (None) -929,935: Return (None) -936,937: [ (None) -937,945: Identifier (Str("date_obj")) -945,946: , (None) -947,949: Identifier (Str("pd")) -949,950: . (None) -950,964: Identifier (Str("json_normalize")) -964,965: ( (None) -965,969: Identifier (Str("data")) -969,970: [ (None) -970,984: StringLiteral (Str("\"tradeHistory\"")) -984,985: ] (None) -985,986: ) (None) -986,987: ] (None) -987,988: NewLine (None) -988,988: Dedent (Indent(3)) -988,988: Dedent (None) -988,988: Dedent (None) -988,994: Except (None) -995,996: ( (None) -996,1003: Identifier (Str("aiohttp")) -1003,1004: . (None) -1004,1015: Identifier (Str("ClientError")) -1015,1016: , (None) -1017,1024: Identifier (Str("asyncio")) -1024,1025: . (None) -1025,1037: Identifier (Str("TimeoutError")) -1037,1038: ) (None) -1038,1039: : (None) -1039,1040: NewLine (None) -1040,1044: Indent (Indent(1)) -1044,1050: Identifier (Str("logger")) -1050,1051: . (None) -1051,1056: Identifier (Str("error")) -1056,1057: ( (None) -1057,1059: FStringStart (Str("f\"")) -1059,1078: FstringMiddle (Str("Request failed for ")) -1078,1079: { (None) -1079,1087: Identifier (Str("date_obj")) -1087,1088: } (None) -1088,1101: FstringMiddle (Str(". Retrying...")) -1101,1102: FStringEnd (Str("\"")) -1102,1103: ) (None) -1103,1104: NewLine (None) -1108,1119: Identifier (Str("retry_count")) -1120,1122: += (None) -1123,1124: Integer (Number("1")) -1124,1125: NewLine (None) -1129,1134: Await (None) -1135,1142: Identifier (Str("asyncio")) -1142,1143: . (None) -1143,1148: Identifier (Str("sleep")) -1148,1149: ( (None) -1149,1150: Integer (Number("1")) -1150,1151: ) (None) -1151,1152: NewLine (None) -1152,1152: Dedent (Indent(1)) +697,713: Identifier (Str("raise_for_status")) +713,714: ( (None) +714,715: ) (None) +715,716: NewLine (None) +728,732: Identifier (Str("data")) +733,734: = (None) +735,740: Await (None) +741,749: Identifier (Str("response")) +749,750: . (None) +750,754: Identifier (Str("json")) +754,755: ( (None) +755,756: ) (None) +756,757: NewLine (None) +769,775: Identifier (Str("logger")) +775,776: . (None) +776,780: Identifier (Str("info")) +780,781: ( (None) +781,783: FStringStart (Str("f\"")) +783,821: FstringMiddle (Str("Successfully fetched trade details on ")) +821,822: { (None) +822,830: Identifier (Str("date_obj")) +830,831: } (None) +831,840: FstringMiddle (Str(" from tse")) +840,841: FStringEnd (Str("\"")) +841,842: ) (None) +842,843: NewLine (None) +855,861: Return (None) +862,863: [ (None) +863,871: Identifier (Str("date_obj")) +871,872: , (None) +873,875: Identifier (Str("pd")) +875,876: . (None) +876,890: Identifier (Str("json_normalize")) +890,891: ( (None) +891,895: Identifier (Str("data")) +895,896: [ (None) +896,910: StringLiteral (Str("\"tradeHistory\"")) +910,911: ] (None) +911,912: ) (None) +912,913: ] (None) +913,914: NewLine (None) +914,914: Dedent (Indent(3)) +914,914: Dedent (None) +914,914: Dedent (None) +914,920: Except (None) +921,922: ( (None) +922,929: Identifier (Str("aiohttp")) +929,930: . (None) +930,941: Identifier (Str("ClientError")) +941,942: , (None) +943,950: Identifier (Str("asyncio")) +950,951: . (None) +951,963: Identifier (Str("TimeoutError")) +963,964: ) (None) +964,965: : (None) +965,966: NewLine (None) +966,970: Indent (Indent(1)) +970,976: Identifier (Str("logger")) +976,977: . (None) +977,982: Identifier (Str("error")) +982,983: ( (None) +983,985: FStringStart (Str("f\"")) +985,1004: FstringMiddle (Str("Request failed for ")) +1004,1005: { (None) +1005,1013: Identifier (Str("date_obj")) +1013,1014: } (None) +1014,1027: FstringMiddle (Str(". Retrying...")) +1027,1028: FStringEnd (Str("\"")) +1028,1029: ) (None) +1029,1030: NewLine (None) +1034,1045: Identifier (Str("retry_count")) +1046,1048: += (None) +1049,1050: Integer (Number("1")) +1050,1051: NewLine (None) +1055,1060: Await (None) +1061,1068: Identifier (Str("asyncio")) +1068,1069: . (None) +1069,1074: Identifier (Str("sleep")) +1074,1075: ( (None) +1075,1076: Integer (Number("1")) +1076,1077: ) (None) +1077,1078: NewLine (None) +1078,1078: Dedent (Indent(1)) diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__string.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__string.snap index 8a21b3c3..2a28331e 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__string.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__string.snap @@ -1,11 +1,11 @@ --- source: parser/src/parser/parser.rs -description: "x = \"simple text\\n\"\n\nraw = r\"raw_text\\n\"\n\ny = \"\"\"multi\"\"\"\n\n# TODO: enable with better error handling in parser\n# raw_err = r'ss\n\nf\"{self.__class__.__name__}({self._display()!r})\"\n\nf\"{num:0.0f}{unit}\"\n\n# f\"some words {a+b:.3f} more words {c+d=} final words\"\n\nf\"tuple argument {name[12:]}\"\n\n# f\"{f\"{f\"{f\"{f\"{f\"{1+1}\"}\"}\"}\"}\"}\"\n\n# f\"\"\"{f'''{f'{f\"{1+1}\"}'}'''}\"\"\"\n" +description: "x = \"simple text\\n\"\n\nraw = r\"raw_text\\n\"\n\ny = \"\"\"multi\"\"\"\n\nf\"{self.__class__.__name__}({self._display()!r})\"\n\nf\"{num:0.0f}{unit}\"\n\nf\"tuple argument {name[12:]}\"\n\nf\"some words {a+b:.3f} more words {c+d=} final words\"\n\nf\"{f\"{f\"{f\"{f\"{f\"{1+1}\"}\"}\"}\"}\"}\"\n\nf\"\"\"{f'''{f'{f\"{1+1}\"}'}'''}\"\"\"\n" --- Module { node: Node { start: 0, - end: 360, + end: 284, }, body: [ AssignStatement( @@ -96,33 +96,33 @@ Module { JoinedStr( JoinedStr { node: Node { - start: 129, - end: 178, + start: 59, + end: 108, }, values: [ FormattedValue( FormattedValue { node: Node { - start: 131, - end: 156, + start: 61, + end: 86, }, value: Attribute( Attribute { node: Node { - start: 132, - end: 155, + start: 62, + end: 85, }, value: Attribute( Attribute { node: Node { - start: 132, - end: 146, + start: 62, + end: 76, }, value: Name( Name { node: Node { - start: 132, - end: 136, + start: 62, + end: 66, }, id: "self", }, @@ -140,8 +140,8 @@ Module { Constant( Constant { node: Node { - start: 156, - end: 157, + start: 86, + end: 87, }, value: "(", }, @@ -149,26 +149,26 @@ Module { FormattedValue( FormattedValue { node: Node { - start: 157, - end: 176, + start: 87, + end: 106, }, value: Call( Call { node: Node { - start: 158, - end: 173, + start: 88, + end: 103, }, func: Attribute( Attribute { node: Node { - start: 158, - end: 171, + start: 88, + end: 101, }, value: Name( Name { node: Node { - start: 158, - end: 162, + start: 88, + end: 92, }, id: "self", }, @@ -189,8 +189,8 @@ Module { Constant( Constant { node: Node { - start: 176, - end: 177, + start: 106, + end: 107, }, value: ")", }, @@ -203,21 +203,21 @@ Module { JoinedStr( JoinedStr { node: Node { - start: 180, - end: 199, + start: 110, + end: 129, }, values: [ FormattedValue( FormattedValue { node: Node { - start: 182, - end: 192, + start: 112, + end: 122, }, value: Name( Name { node: Node { - start: 183, - end: 186, + start: 113, + end: 116, }, id: "num", }, @@ -227,15 +227,15 @@ Module { JoinedStr( JoinedStr { node: Node { - start: 186, - end: 191, + start: 116, + end: 121, }, values: [ Constant( Constant { node: Node { - start: 187, - end: 191, + start: 117, + end: 121, }, value: "0.0f", }, @@ -249,14 +249,14 @@ Module { FormattedValue( FormattedValue { node: Node { - start: 192, - end: 198, + start: 122, + end: 128, }, value: Name( Name { node: Node { - start: 193, - end: 197, + start: 123, + end: 127, }, id: "unit", }, @@ -273,15 +273,15 @@ Module { JoinedStr( JoinedStr { node: Node { - start: 258, - end: 287, + start: 131, + end: 160, }, values: [ Constant( Constant { node: Node { - start: 260, - end: 275, + start: 133, + end: 148, }, value: "tuple argument ", }, @@ -289,20 +289,20 @@ Module { FormattedValue( FormattedValue { node: Node { - start: 275, - end: 286, + start: 148, + end: 159, }, value: Subscript( Subscript { node: Node { - start: 276, - end: 285, + start: 149, + end: 158, }, value: Name( Name { node: Node { - start: 276, - end: 280, + start: 149, + end: 153, }, id: "name", }, @@ -310,15 +310,15 @@ Module { slice: Slice( Slice { node: Node { - start: 281, - end: 284, + start: 154, + end: 157, }, lower: Some( Constant( Constant { node: Node { - start: 281, - end: 283, + start: 154, + end: 156, }, value: 12, }, @@ -338,5 +338,396 @@ Module { }, ), ), + ExpressionStatement( + JoinedStr( + JoinedStr { + node: Node { + start: 162, + end: 215, + }, + values: [ + Constant( + Constant { + node: Node { + start: 164, + end: 175, + }, + value: "some words ", + }, + ), + FormattedValue( + FormattedValue { + node: Node { + start: 175, + end: 184, + }, + value: BinOp( + BinOp { + node: Node { + start: 176, + end: 179, + }, + op: Add, + left: Name( + Name { + node: Node { + start: 176, + end: 177, + }, + id: "a", + }, + ), + right: Name( + Name { + node: Node { + start: 178, + end: 179, + }, + id: "b", + }, + ), + }, + ), + conversion: -1, + format_spec: Some( + JoinedStr( + JoinedStr { + node: Node { + start: 179, + end: 183, + }, + values: [ + Constant( + Constant { + node: Node { + start: 180, + end: 183, + }, + value: ".3f", + }, + ), + ], + }, + ), + ), + }, + ), + Constant( + Constant { + node: Node { + start: 184, + end: 196, + }, + value: " more words ", + }, + ), + FormattedValue( + FormattedValue { + node: Node { + start: 196, + end: 202, + }, + value: BinOp( + BinOp { + node: Node { + start: 197, + end: 200, + }, + op: Add, + left: Name( + Name { + node: Node { + start: 197, + end: 198, + }, + id: "c", + }, + ), + right: Name( + Name { + node: Node { + start: 199, + end: 200, + }, + id: "d", + }, + ), + }, + ), + conversion: 114, + format_spec: None, + }, + ), + Constant( + Constant { + node: Node { + start: 202, + end: 214, + }, + value: " final words", + }, + ), + ], + }, + ), + ), + ExpressionStatement( + JoinedStr( + JoinedStr { + node: Node { + start: 217, + end: 250, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 219, + end: 249, + }, + value: JoinedStr( + JoinedStr { + node: Node { + start: 220, + end: 248, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 222, + end: 247, + }, + value: JoinedStr( + JoinedStr { + node: Node { + start: 223, + end: 246, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 225, + end: 245, + }, + value: JoinedStr( + JoinedStr { + node: Node { + start: 226, + end: 244, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 228, + end: 243, + }, + value: JoinedStr( + JoinedStr { + node: Node { + start: 229, + end: 242, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 231, + end: 241, + }, + value: JoinedStr( + JoinedStr { + node: Node { + start: 232, + end: 240, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 234, + end: 239, + }, + value: BinOp( + BinOp { + node: Node { + start: 235, + end: 238, + }, + op: Add, + left: Constant( + Constant { + node: Node { + start: 235, + end: 236, + }, + value: 1, + }, + ), + right: Constant( + Constant { + node: Node { + start: 237, + end: 238, + }, + value: 1, + }, + ), + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + ), + ExpressionStatement( + JoinedStr( + JoinedStr { + node: Node { + start: 252, + end: 283, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 256, + end: 280, + }, + value: JoinedStr( + JoinedStr { + node: Node { + start: 257, + end: 279, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 261, + end: 276, + }, + value: JoinedStr( + JoinedStr { + node: Node { + start: 262, + end: 275, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 264, + end: 274, + }, + value: JoinedStr( + JoinedStr { + node: Node { + start: 265, + end: 273, + }, + values: [ + FormattedValue( + FormattedValue { + node: Node { + start: 267, + end: 272, + }, + value: BinOp( + BinOp { + node: Node { + start: 268, + end: 271, + }, + op: Add, + left: Constant( + Constant { + node: Node { + start: 268, + end: 269, + }, + value: 1, + }, + ), + right: Constant( + Constant { + node: Node { + start: 270, + end: 271, + }, + value: 1, + }, + ), + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + conversion: -1, + format_spec: None, + }, + ), + ], + }, + ), + ), ], } diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try.snap index eb20940c..9e38a19c 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try.snap @@ -1,11 +1,11 @@ --- source: parser/src/parser/parser.rs -description: "# in some cases last price or adj price is undefined\ntry:\n last_price = int(price_section[2])\n# when instead of number value is `F`\nexcept (ValueError, IndexError):\n last_price = None\ntry:\n adj_close = int(price_section[3])\nexcept (ValueError, IndexError):\n adj_close = None\ntry:\n market_cap = adj_close * self.total_shares\nexcept ValueError:\n market_cap = None\n\n\ntry:\n async with session.get(\n url, headers=TRADE_DETAILS_HEADER, timeout=100\n ) as response:\n if response.status == 503:\n logger.info(\n f\"Received 503 Service Unavailable on {date_obj}. Retrying...\"\n )\n retry_count += 1\n await asyncio.sleep(1)\n else:\n response.raise_for_status()\n data = await response.json()\n logger.info(\n f\"Successfully fetched trade details on {date_obj} from tse\"\n )\n return [date_obj, pd.json_normalize(data[\"tradeHistory\"])]\nexcept (aiohttp.ClientError, asyncio.TimeoutError):\n logger.error(f\"Request failed for {date_obj}. Retrying...\")\n retry_count += 1\n await asyncio.sleep(1)\n" +description: "# in some cases last price or adj price is undefined\ntry:\n last_price = int(price_section[2])\n# when instead of number value is `F`\nexcept (ValueError, IndexError):\n last_price = None\ntry:\n adj_close = int(price_section[3])\nexcept (ValueError, IndexError):\n adj_close = None\ntry:\n market_cap = adj_close * self.total_shares\nexcept ValueError:\n market_cap = None\n\n\ntry:\n async with session.get(url, headers=TRADE_DETAILS_HEADER, timeout=100) as response:\n if response.status == 503:\n logger.info(f\"Received 503 Service Unavailable on {date_obj}. Retrying...\")\n retry_count += 1\n await asyncio.sleep(1)\n else:\n response.raise_for_status()\n data = await response.json()\n logger.info(f\"Successfully fetched trade details on {date_obj} from tse\")\n return [date_obj, pd.json_normalize(data[\"tradeHistory\"])]\nexcept (aiohttp.ClientError, asyncio.TimeoutError):\n logger.error(f\"Request failed for {date_obj}. Retrying...\")\n retry_count += 1\n await asyncio.sleep(1)\n" --- Module { node: Node { start: 0, - end: 1152, + end: 1078, }, body: [ TryStatement( @@ -425,26 +425,26 @@ Module { Try { node: Node { start: 382, - end: 1152, + end: 1078, }, body: [ AsyncWithStatement( AsyncWith { node: Node { start: 391, - end: 988, + end: 914, }, items: [ WithItem { node: Node { start: 402, - end: 487, + end: 473, }, context_expr: Call( Call { node: Node { start: 402, - end: 475, + end: 461, }, func: Attribute( Attribute { @@ -468,8 +468,8 @@ Module { Name( Name { node: Node { - start: 423, - end: 426, + start: 414, + end: 417, }, id: "url", }, @@ -478,8 +478,8 @@ Module { keywords: [ Keyword { node: Node { - start: 428, - end: 456, + start: 419, + end: 447, }, arg: Some( "headers", @@ -487,8 +487,8 @@ Module { value: Name( Name { node: Node { - start: 436, - end: 456, + start: 427, + end: 447, }, id: "TRADE_DETAILS_HEADER", }, @@ -496,8 +496,8 @@ Module { }, Keyword { node: Node { - start: 458, - end: 470, + start: 449, + end: 460, }, arg: Some( "timeout", @@ -505,8 +505,8 @@ Module { value: Constant( Constant { node: Node { - start: 466, - end: 470, + start: 457, + end: 460, }, value: 100, }, @@ -521,8 +521,8 @@ Module { Name( Name { node: Node { - start: 479, - end: 487, + start: 465, + end: 473, }, id: "response", }, @@ -534,26 +534,26 @@ Module { IfStatement( If { node: Node { - start: 497, - end: 988, + start: 483, + end: 914, }, test: Compare( Compare { node: Node { - start: 500, - end: 522, + start: 486, + end: 508, }, left: Attribute( Attribute { node: Node { - start: 500, - end: 515, + start: 486, + end: 501, }, value: Name( Name { node: Node { - start: 500, - end: 508, + start: 486, + end: 494, }, id: "response", }, @@ -568,8 +568,8 @@ Module { Constant( Constant { node: Node { - start: 519, - end: 522, + start: 505, + end: 508, }, value: 503, }, @@ -582,20 +582,20 @@ Module { Call( Call { node: Node { - start: 536, - end: 641, + start: 522, + end: 597, }, func: Attribute( Attribute { node: Node { - start: 536, - end: 547, + start: 522, + end: 533, }, value: Name( Name { node: Node { - start: 536, - end: 542, + start: 522, + end: 528, }, id: "logger", }, @@ -607,15 +607,15 @@ Module { JoinedStr( JoinedStr { node: Node { - start: 565, - end: 628, + start: 534, + end: 596, }, values: [ Constant( Constant { node: Node { - start: 567, - end: 603, + start: 536, + end: 572, }, value: "Received 503 Service Unavailable on ", }, @@ -623,14 +623,14 @@ Module { FormattedValue( FormattedValue { node: Node { - start: 603, - end: 613, + start: 572, + end: 582, }, value: Name( Name { node: Node { - start: 604, - end: 612, + start: 573, + end: 581, }, id: "date_obj", }, @@ -642,8 +642,8 @@ Module { Constant( Constant { node: Node { - start: 613, - end: 626, + start: 582, + end: 595, }, value: ". Retrying...", }, @@ -661,14 +661,14 @@ Module { AugAssignStatement( AugAssign { node: Node { - start: 654, - end: 670, + start: 610, + end: 626, }, target: Name( Name { node: Node { - start: 654, - end: 665, + start: 610, + end: 621, }, id: "retry_count", }, @@ -677,8 +677,8 @@ Module { value: Constant( Constant { node: Node { - start: 669, - end: 670, + start: 625, + end: 626, }, value: 1, }, @@ -689,26 +689,26 @@ Module { Await( Await { node: Node { - start: 683, - end: 705, + start: 639, + end: 661, }, value: Call( Call { node: Node { - start: 689, - end: 705, + start: 645, + end: 661, }, func: Attribute( Attribute { node: Node { - start: 689, - end: 702, + start: 645, + end: 658, }, value: Name( Name { node: Node { - start: 689, - end: 696, + start: 645, + end: 652, }, id: "asyncio", }, @@ -720,8 +720,8 @@ Module { Constant( Constant { node: Node { - start: 703, - end: 704, + start: 659, + end: 660, }, value: 1, }, @@ -741,20 +741,20 @@ Module { Call( Call { node: Node { - start: 732, - end: 759, + start: 688, + end: 715, }, func: Attribute( Attribute { node: Node { - start: 732, - end: 757, + start: 688, + end: 713, }, value: Name( Name { node: Node { - start: 732, - end: 740, + start: 688, + end: 696, }, id: "response", }, @@ -772,15 +772,15 @@ Module { AssignStatement( Assign { node: Node { - start: 772, - end: 800, + start: 728, + end: 756, }, targets: [ Name( Name { node: Node { - start: 772, - end: 776, + start: 728, + end: 732, }, id: "data", }, @@ -789,26 +789,26 @@ Module { value: Await( Await { node: Node { - start: 779, - end: 800, + start: 735, + end: 756, }, value: Call( Call { node: Node { - start: 785, - end: 800, + start: 741, + end: 756, }, func: Attribute( Attribute { node: Node { - start: 785, - end: 798, + start: 741, + end: 754, }, value: Name( Name { node: Node { - start: 785, - end: 793, + start: 741, + end: 749, }, id: "response", }, @@ -830,20 +830,20 @@ Module { Call( Call { node: Node { - start: 813, - end: 916, + start: 769, + end: 842, }, func: Attribute( Attribute { node: Node { - start: 813, - end: 824, + start: 769, + end: 780, }, value: Name( Name { node: Node { - start: 813, - end: 819, + start: 769, + end: 775, }, id: "logger", }, @@ -855,15 +855,15 @@ Module { JoinedStr( JoinedStr { node: Node { - start: 842, - end: 903, + start: 781, + end: 841, }, values: [ Constant( Constant { node: Node { - start: 844, - end: 882, + start: 783, + end: 821, }, value: "Successfully fetched trade details on ", }, @@ -871,14 +871,14 @@ Module { FormattedValue( FormattedValue { node: Node { - start: 882, - end: 892, + start: 821, + end: 831, }, value: Name( Name { node: Node { - start: 883, - end: 891, + start: 822, + end: 830, }, id: "date_obj", }, @@ -890,8 +890,8 @@ Module { Constant( Constant { node: Node { - start: 892, - end: 901, + start: 831, + end: 840, }, value: " from tse", }, @@ -909,22 +909,22 @@ Module { Return( Return { node: Node { - start: 929, - end: 987, + start: 855, + end: 913, }, value: Some( List( List { node: Node { - start: 936, - end: 987, + start: 862, + end: 913, }, elements: [ Name( Name { node: Node { - start: 937, - end: 945, + start: 863, + end: 871, }, id: "date_obj", }, @@ -932,20 +932,20 @@ Module { Call( Call { node: Node { - start: 947, - end: 986, + start: 873, + end: 912, }, func: Attribute( Attribute { node: Node { - start: 947, - end: 964, + start: 873, + end: 890, }, value: Name( Name { node: Node { - start: 947, - end: 949, + start: 873, + end: 875, }, id: "pd", }, @@ -957,14 +957,14 @@ Module { Subscript( Subscript { node: Node { - start: 965, - end: 985, + start: 891, + end: 911, }, value: Name( Name { node: Node { - start: 965, - end: 969, + start: 891, + end: 895, }, id: "data", }, @@ -972,8 +972,8 @@ Module { slice: Constant( Constant { node: Node { - start: 970, - end: 984, + start: 896, + end: 910, }, value: "tradeHistory", }, @@ -1002,28 +1002,28 @@ Module { handlers: [ ExceptHandler { node: Node { - start: 988, - end: 1152, + start: 914, + end: 1078, }, typ: Some( Tuple( Tuple { node: Node { - start: 995, - end: 1037, + start: 921, + end: 963, }, elements: [ Attribute( Attribute { node: Node { - start: 996, - end: 1015, + start: 922, + end: 941, }, value: Name( Name { node: Node { - start: 996, - end: 1003, + start: 922, + end: 929, }, id: "aiohttp", }, @@ -1034,14 +1034,14 @@ Module { Attribute( Attribute { node: Node { - start: 1017, - end: 1037, + start: 943, + end: 963, }, value: Name( Name { node: Node { - start: 1017, - end: 1024, + start: 943, + end: 950, }, id: "asyncio", }, @@ -1059,20 +1059,20 @@ Module { Call( Call { node: Node { - start: 1044, - end: 1103, + start: 970, + end: 1029, }, func: Attribute( Attribute { node: Node { - start: 1044, - end: 1056, + start: 970, + end: 982, }, value: Name( Name { node: Node { - start: 1044, - end: 1050, + start: 970, + end: 976, }, id: "logger", }, @@ -1084,15 +1084,15 @@ Module { JoinedStr( JoinedStr { node: Node { - start: 1057, - end: 1102, + start: 983, + end: 1028, }, values: [ Constant( Constant { node: Node { - start: 1059, - end: 1078, + start: 985, + end: 1004, }, value: "Request failed for ", }, @@ -1100,14 +1100,14 @@ Module { FormattedValue( FormattedValue { node: Node { - start: 1078, - end: 1088, + start: 1004, + end: 1014, }, value: Name( Name { node: Node { - start: 1079, - end: 1087, + start: 1005, + end: 1013, }, id: "date_obj", }, @@ -1119,8 +1119,8 @@ Module { Constant( Constant { node: Node { - start: 1088, - end: 1101, + start: 1014, + end: 1027, }, value: ". Retrying...", }, @@ -1138,14 +1138,14 @@ Module { AugAssignStatement( AugAssign { node: Node { - start: 1108, - end: 1124, + start: 1034, + end: 1050, }, target: Name( Name { node: Node { - start: 1108, - end: 1119, + start: 1034, + end: 1045, }, id: "retry_count", }, @@ -1154,8 +1154,8 @@ Module { value: Constant( Constant { node: Node { - start: 1123, - end: 1124, + start: 1049, + end: 1050, }, value: 1, }, @@ -1166,26 +1166,26 @@ Module { Await( Await { node: Node { - start: 1129, - end: 1151, + start: 1055, + end: 1077, }, value: Call( Call { node: Node { - start: 1135, - end: 1151, + start: 1061, + end: 1077, }, func: Attribute( Attribute { node: Node { - start: 1135, - end: 1148, + start: 1061, + end: 1074, }, value: Name( Name { node: Node { - start: 1135, - end: 1142, + start: 1061, + end: 1068, }, id: "asyncio", }, @@ -1197,8 +1197,8 @@ Module { Constant( Constant { node: Node { - start: 1149, - end: 1150, + start: 1075, + end: 1076, }, value: 1, },