From 842911396f8db6091d927054a5dc54d849d855c0 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Sun, 1 Oct 2023 23:15:03 +0200 Subject: [PATCH] Fix consecutive binary expressions --- parser/src/parser/parser.rs | 21 +- ...er__parser__parser__tests__comment-10.snap | 2 +- ...ser__parser__parser__tests__comment-8.snap | 2 +- ...ser__parser__parser__tests__comment-9.snap | 2 +- ..._parser__tests__complete@functions.py.snap | 914 +++++++++++++++++- ...ser__tests__complete@input_program.py.snap | 10 +- ...ts__complete@separate_statements.py-2.snap | 4 +- ...arser__parser__tests__complete@try.py.snap | 430 ++++++++ ...r__tests__one_liners@from_import.py-2.snap | 2 +- ..._parser__tests__one_liners@lists.py-7.snap | 4 +- parser/test_data/inputs/functions.py | 14 + parser/test_data/inputs/try.py | 15 + 12 files changed, 1395 insertions(+), 25 deletions(-) create mode 100644 parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@try.py.snap create mode 100644 parser/test_data/inputs/try.py diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 8e13f62a..4aaf0187 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -172,6 +172,9 @@ impl Parser { Ok(token) => { self.prev_token_end = self.cur_token.end; self.cur_token = token; + if self.cur_kind() == Kind::Comment { + self.bump(Kind::Comment); + } } } } @@ -591,7 +594,7 @@ impl Parser { fn parse_try_statement(&mut self) -> Result { let node = self.start_node(); let mut is_try_star = false; - self.bump(Kind::Try); + self.expect(Kind::Try); self.expect(Kind::Colon)?; let body = self.parse_suite()?; let handlers = if self.at(Kind::Except) { @@ -725,7 +728,7 @@ impl Parser { self.bump(Kind::NewLine); } - if self.at(Kind::Def) { + if self.at(Kind::Def) || self.at(Kind::Async) { self.parse_function_definition(decorators) } else { self.parse_class_definition(decorators) @@ -2205,16 +2208,16 @@ impl Parser { // https://docs.python.org/3/reference/expressions.html#binary-arithmetic-operations fn parse_binary_arithmetic_operation(&mut self) -> Result { let node = self.start_node(); - let lhs = self.parse_unary_arithmetric_operation()?; - if is_bin_arithmetic_op(&self.cur_kind()) { + let mut lhs = self.parse_unary_arithmetric_operation()?; + while is_bin_arithmetic_op(&self.cur_kind()) { let op = self.parse_bin_arithmetic_op()?; let rhs = self.parse_unary_arithmetric_operation()?; - return Ok(Expression::BinOp(Box::new(BinOp { + lhs = Expression::BinOp(Box::new(BinOp { node: self.finish_node(node), op, left: Box::new(lhs), right: Box::new(rhs), - }))); + })); } Ok(lhs) } @@ -2270,7 +2273,6 @@ impl Parser { } else if is_atom(&self.cur_kind()) { self.parse_atom()? } else { - panic!("invalid primary expression {:?}", self.cur_kind()); return Err(self.unepxted_token(node, self.cur_kind()).err().unwrap()); }; @@ -2281,6 +2283,7 @@ impl Parser { // https://docs.python.org/3/reference/expressions.html#slicings self.parse_subscript(node, atom_or_primary) } else if self.eat(Kind::LeftParen) { + self.bump(Kind::NewLine); // parse call // https://docs.python.org/3/reference/expressions.html#calls let mut positional_args = vec![]; @@ -3683,7 +3686,7 @@ class a: pass", fn test_complete() { glob!("../../test_data", "inputs/*.py", |path| { let test_case = fs::read_to_string(path).unwrap(); - let mut parser = Parser::new(test_case.clone(), String::from("")); + let mut parser = Parser::new(test_case.clone(), String::from(path.to_str().unwrap_or(""))); let program = parser.parse(); insta::with_settings!({ @@ -3709,7 +3712,7 @@ class a: pass", glob!("../../test_data", "inputs/one_liners/*.py", |path| { let input = fs::read_to_string(path).unwrap(); for test_case in input.split("\n\n") { - let mut parser = Parser::new(test_case.to_string(), String::from("")); + let mut parser = Parser::new(test_case.to_string(), String::from(path.to_str().unwrap_or(""))); let program = parser.parse(); insta::with_settings!({ diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-10.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-10.snap index 858dc4d1..4bcbea82 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-10.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-10.snap @@ -34,7 +34,7 @@ Module { Constant { node: Node { start: 13, - end: 16, + end: 20, }, value: Ellipsis, }, diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-8.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-8.snap index 5144841f..5558e7f5 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-8.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-8.snap @@ -34,7 +34,7 @@ Module { Constant { node: Node { start: 9, - end: 12, + end: 16, }, value: Ellipsis, }, diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-9.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-9.snap index 13797ad2..7b9df0ba 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-9.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__comment-9.snap @@ -34,7 +34,7 @@ Module { Constant { node: Node { start: 9, - end: 12, + end: 16, }, value: Ellipsis, }, diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@functions.py.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@functions.py.snap index 3c683fea..228a6c1f 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@functions.py.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@functions.py.snap @@ -1,12 +1,12 @@ --- source: parser/src/parser/parser.rs -description: "def _handle_ticker_index(symbol):\n ticker_index = symbols_data.get_ticker_index(symbol)\n\n if ticker_index is None:\n market_symbol = get_symbol_info(symbol)\n if market_symbol is not None:\n symbols_data.append_symbol_to_file(market_symbol)\n ticker_index = market_symbol.index\n return ticker_index\n\n\ndef _extract_ticker_client_types_data(ticker_index: str) -> List:\n url = TSE_CLIENT_TYPE_DATA_URL.format(ticker_index)\n with requests_retry_session() as session:\n response = session.get(url, timeout=5)\n data = response.text.split(';')\n return data\n" +description: "def _handle_ticker_index(symbol):\n ticker_index = symbols_data.get_ticker_index(symbol)\n\n if ticker_index is None:\n market_symbol = get_symbol_info(symbol)\n if market_symbol is not None:\n symbols_data.append_symbol_to_file(market_symbol)\n ticker_index = market_symbol.index\n return ticker_index\n\n\ndef _extract_ticker_client_types_data(ticker_index: str) -> List:\n url = TSE_CLIENT_TYPE_DATA_URL.format(ticker_index)\n with requests_retry_session() as session:\n response = session.get(url, timeout=5)\n data = response.text.split(';')\n return data\n\ndef common_process(df: pd.DataFrame, date: str):\n if len(df) == 0:\n return pd.DataFrame(columns=list(api_to_orderbook_mapping.keys()))\n df.rename(columns=reversed_keys, inplace=True)\n df = df.loc[:, list(api_to_orderbook_mapping.keys())]\n df[\"datetime\"] = pd.to_datetime(\n date + \" \" + df[\"datetime\"].astype(str), format=\"%Y%m%d %H%M%S\"\n )\n df = df.sort_values([\"datetime\", \"depth\"], ascending=[True, True])\n df.set_index(\"datetime\", inplace=True)\n df.drop(columns=[\"refID\"], inplace=True)\n return df\n\n" input_file: parser/test_data/inputs/functions.py --- Module { node: Node { start: 0, - end: 609, + end: 1153, }, body: [ FunctionDef( @@ -340,7 +340,7 @@ Module { FunctionDef { node: Node { start: 342, - end: 609, + end: 610, }, name: "_extract_ticker_client_types_data", args: Arguments { @@ -670,5 +670,913 @@ Module { type_comment: None, }, ), + FunctionDef( + FunctionDef { + node: Node { + start: 610, + end: 1153, + }, + name: "common_process", + args: Arguments { + node: Node { + start: 629, + end: 656, + }, + posonlyargs: [], + args: [ + Arg { + node: Node { + start: 629, + end: 645, + }, + arg: "df", + annotation: Some( + Attribute( + Attribute { + node: Node { + start: 633, + end: 645, + }, + value: Name( + Name { + node: Node { + start: 633, + end: 635, + }, + id: "pd", + }, + ), + attr: "DataFrame", + }, + ), + ), + }, + Arg { + node: Node { + start: 647, + end: 656, + }, + arg: "date", + annotation: Some( + Name( + Name { + node: Node { + start: 653, + end: 656, + }, + id: "str", + }, + ), + ), + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: [ + IfStatement( + If { + node: Node { + start: 663, + end: 759, + }, + test: Compare( + Compare { + node: Node { + start: 666, + end: 678, + }, + left: Call( + Call { + node: Node { + start: 666, + end: 673, + }, + func: Name( + Name { + node: Node { + start: 666, + end: 669, + }, + id: "len", + }, + ), + args: [ + Name( + Name { + node: Node { + start: 670, + end: 672, + }, + id: "df", + }, + ), + ], + keywords: [], + starargs: None, + kwargs: None, + }, + ), + ops: [ + Eq, + ], + comparators: [ + Constant( + Constant { + node: Node { + start: 677, + end: 678, + }, + value: Int( + "0", + ), + }, + ), + ], + }, + ), + body: [ + Return( + Return { + node: Node { + start: 688, + end: 754, + }, + value: Some( + Call( + Call { + node: Node { + start: 707, + end: 754, + }, + func: Attribute( + Attribute { + node: Node { + start: 695, + end: 707, + }, + value: Name( + Name { + node: Node { + start: 695, + end: 697, + }, + id: "pd", + }, + ), + attr: "DataFrame", + }, + ), + args: [], + keywords: [ + Keyword { + node: Node { + start: 708, + end: 753, + }, + arg: Some( + "columns", + ), + value: Call( + Call { + node: Node { + start: 716, + end: 753, + }, + func: Name( + Name { + node: Node { + start: 716, + end: 720, + }, + id: "list", + }, + ), + args: [ + Call( + Call { + node: Node { + start: 750, + end: 752, + }, + func: Attribute( + Attribute { + node: Node { + start: 721, + end: 750, + }, + value: Name( + Name { + node: Node { + start: 721, + end: 745, + }, + id: "api_to_orderbook_mapping", + }, + ), + attr: "keys", + }, + ), + args: [], + keywords: [], + starargs: None, + kwargs: None, + }, + ), + ], + keywords: [], + starargs: None, + kwargs: None, + }, + ), + }, + ], + starargs: None, + kwargs: None, + }, + ), + ), + }, + ), + ], + orelse: [], + }, + ), + ExpressionStatement( + Call( + Call { + node: Node { + start: 768, + end: 805, + }, + func: Attribute( + Attribute { + node: Node { + start: 759, + end: 768, + }, + value: Name( + Name { + node: Node { + start: 759, + end: 761, + }, + id: "df", + }, + ), + attr: "rename", + }, + ), + args: [], + keywords: [ + Keyword { + node: Node { + start: 769, + end: 790, + }, + arg: Some( + "columns", + ), + value: Name( + Name { + node: Node { + start: 777, + end: 790, + }, + id: "reversed_keys", + }, + ), + }, + Keyword { + node: Node { + start: 792, + end: 804, + }, + arg: Some( + "inplace", + ), + value: Constant( + Constant { + node: Node { + start: 800, + end: 804, + }, + value: Bool( + true, + ), + }, + ), + }, + ], + starargs: None, + kwargs: None, + }, + ), + ), + AssignStatement( + Assign { + node: Node { + start: 810, + end: 863, + }, + targets: [ + Name( + Name { + node: Node { + start: 810, + end: 812, + }, + id: "df", + }, + ), + ], + value: Subscript( + Subscript { + node: Node { + start: 821, + end: 863, + }, + value: Attribute( + Attribute { + node: Node { + start: 815, + end: 821, + }, + value: Name( + Name { + node: Node { + start: 815, + end: 817, + }, + id: "df", + }, + ), + attr: "loc", + }, + ), + slice: Tuple( + Tuple { + node: Node { + start: 822, + end: 863, + }, + elements: [ + Slice( + Slice { + node: Node { + start: 822, + end: 823, + }, + lower: None, + upper: None, + step: None, + }, + ), + Call( + Call { + node: Node { + start: 825, + end: 862, + }, + func: Name( + Name { + node: Node { + start: 825, + end: 829, + }, + id: "list", + }, + ), + args: [ + Call( + Call { + node: Node { + start: 859, + end: 861, + }, + func: Attribute( + Attribute { + node: Node { + start: 830, + end: 859, + }, + value: Name( + Name { + node: Node { + start: 830, + end: 854, + }, + id: "api_to_orderbook_mapping", + }, + ), + attr: "keys", + }, + ), + args: [], + keywords: [], + starargs: None, + kwargs: None, + }, + ), + ], + keywords: [], + starargs: None, + kwargs: None, + }, + ), + ], + }, + ), + }, + ), + }, + ), + AssignStatement( + Assign { + node: Node { + start: 868, + end: 978, + }, + targets: [ + Subscript( + Subscript { + node: Node { + start: 868, + end: 882, + }, + value: Name( + Name { + node: Node { + start: 868, + end: 870, + }, + id: "df", + }, + ), + slice: Constant( + Constant { + node: Node { + start: 871, + end: 881, + }, + value: Str( + "datetime", + ), + }, + ), + }, + ), + ], + value: Call( + Call { + node: Node { + start: 899, + end: 978, + }, + func: Attribute( + Attribute { + node: Node { + start: 885, + end: 899, + }, + value: Name( + Name { + node: Node { + start: 885, + end: 887, + }, + id: "pd", + }, + ), + attr: "to_datetime", + }, + ), + args: [ + BinOp( + BinOp { + node: Node { + start: 909, + end: 948, + }, + op: Add, + left: BinOp( + BinOp { + node: Node { + start: 909, + end: 919, + }, + op: Add, + left: Name( + Name { + node: Node { + start: 909, + end: 913, + }, + id: "date", + }, + ), + right: Constant( + Constant { + node: Node { + start: 916, + end: 919, + }, + value: Str( + " ", + ), + }, + ), + }, + ), + right: Call( + Call { + node: Node { + start: 943, + end: 948, + }, + func: Attribute( + Attribute { + node: Node { + start: 936, + end: 943, + }, + value: Subscript( + Subscript { + node: Node { + start: 922, + end: 936, + }, + value: Name( + Name { + node: Node { + start: 922, + end: 924, + }, + id: "df", + }, + ), + slice: Constant( + Constant { + node: Node { + start: 925, + end: 935, + }, + value: Str( + "datetime", + ), + }, + ), + }, + ), + attr: "astype", + }, + ), + args: [ + Name( + Name { + node: Node { + start: 944, + end: 947, + }, + id: "str", + }, + ), + ], + keywords: [], + starargs: None, + kwargs: None, + }, + ), + }, + ), + ], + keywords: [ + Keyword { + node: Node { + start: 950, + end: 972, + }, + arg: Some( + "format", + ), + value: Constant( + Constant { + node: Node { + start: 957, + end: 972, + }, + value: Str( + "%Y%m%d %H%M%S", + ), + }, + ), + }, + ], + starargs: None, + kwargs: None, + }, + ), + }, + ), + AssignStatement( + Assign { + node: Node { + start: 983, + end: 1049, + }, + targets: [ + Name( + Name { + node: Node { + start: 983, + end: 985, + }, + id: "df", + }, + ), + ], + value: Call( + Call { + node: Node { + start: 1002, + end: 1049, + }, + func: Attribute( + Attribute { + node: Node { + start: 988, + end: 1002, + }, + value: Name( + Name { + node: Node { + start: 988, + end: 990, + }, + id: "df", + }, + ), + attr: "sort_values", + }, + ), + args: [ + List( + List { + node: Node { + start: 1003, + end: 1024, + }, + elements: [ + Constant( + Constant { + node: Node { + start: 1004, + end: 1014, + }, + value: Str( + "datetime", + ), + }, + ), + Constant( + Constant { + node: Node { + start: 1016, + end: 1023, + }, + value: Str( + "depth", + ), + }, + ), + ], + }, + ), + ], + keywords: [ + Keyword { + node: Node { + start: 1026, + end: 1048, + }, + arg: Some( + "ascending", + ), + value: List( + List { + node: Node { + start: 1036, + end: 1048, + }, + elements: [ + Constant( + Constant { + node: Node { + start: 1037, + end: 1041, + }, + value: Bool( + true, + ), + }, + ), + Constant( + Constant { + node: Node { + start: 1043, + end: 1047, + }, + value: Bool( + true, + ), + }, + ), + ], + }, + ), + }, + ], + starargs: None, + kwargs: None, + }, + ), + }, + ), + ExpressionStatement( + Call( + Call { + node: Node { + start: 1066, + end: 1092, + }, + func: Attribute( + Attribute { + node: Node { + start: 1054, + end: 1066, + }, + value: Name( + Name { + node: Node { + start: 1054, + end: 1056, + }, + id: "df", + }, + ), + attr: "set_index", + }, + ), + args: [ + Constant( + Constant { + node: Node { + start: 1067, + end: 1077, + }, + value: Str( + "datetime", + ), + }, + ), + ], + keywords: [ + Keyword { + node: Node { + start: 1079, + end: 1091, + }, + arg: Some( + "inplace", + ), + value: Constant( + Constant { + node: Node { + start: 1087, + end: 1091, + }, + value: Bool( + true, + ), + }, + ), + }, + ], + starargs: None, + kwargs: None, + }, + ), + ), + ExpressionStatement( + Call( + Call { + node: Node { + start: 1104, + end: 1137, + }, + func: Attribute( + Attribute { + node: Node { + start: 1097, + end: 1104, + }, + value: Name( + Name { + node: Node { + start: 1097, + end: 1099, + }, + id: "df", + }, + ), + attr: "drop", + }, + ), + args: [], + keywords: [ + Keyword { + node: Node { + start: 1105, + end: 1122, + }, + arg: Some( + "columns", + ), + value: List( + List { + node: Node { + start: 1113, + end: 1122, + }, + elements: [ + Constant( + Constant { + node: Node { + start: 1114, + end: 1121, + }, + value: Str( + "refID", + ), + }, + ), + ], + }, + ), + }, + Keyword { + node: Node { + start: 1124, + end: 1136, + }, + arg: Some( + "inplace", + ), + value: Constant( + Constant { + node: Node { + start: 1132, + end: 1136, + }, + value: Bool( + true, + ), + }, + ), + }, + ], + starargs: None, + kwargs: None, + }, + ), + ), + Return( + Return { + node: Node { + start: 1142, + end: 1151, + }, + value: Some( + Name( + Name { + node: Node { + start: 1149, + end: 1151, + }, + id: "df", + }, + ), + ), + }, + ), + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + ), ], } diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@input_program.py.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@input_program.py.snap index ee60a55f..48419f6d 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@input_program.py.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@input_program.py.snap @@ -13132,14 +13132,14 @@ Module { Return { node: Node { start: 13922, - end: 13948, + end: 13961, }, value: Some( Subscript( Subscript { node: Node { start: 13929, - end: 13948, + end: 13961, }, value: Name( Name { @@ -14230,7 +14230,7 @@ Module { Assign { node: Node { start: 15031, - end: 15072, + end: 15092, }, targets: [ Attribute( @@ -14256,7 +14256,7 @@ Module { Subscript { node: Node { start: 15053, - end: 15072, + end: 15092, }, value: Name( Name { @@ -14289,7 +14289,7 @@ Module { Call { node: Node { start: 15151, - end: 15172, + end: 15189, }, func: Attribute( Attribute { diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@separate_statements.py-2.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@separate_statements.py-2.snap index 251512a7..ef246509 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@separate_statements.py-2.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@separate_statements.py-2.snap @@ -5,7 +5,7 @@ input_file: parser/test_data/inputs/separate_statements.py --- [ InvalidSyntax { - path: "", + path: "/Users/glyphack/Programming/enderpy/parser/test_data/inputs/separate_statements.py", msg: "Statement does not end in new line or semicolon", line: 8, input: " x = 1", @@ -16,7 +16,7 @@ input_file: parser/test_data/inputs/separate_statements.py ), }, InvalidSyntax { - path: "", + path: "/Users/glyphack/Programming/enderpy/parser/test_data/inputs/separate_statements.py", msg: "Statement does not end in new line or semicolon", line: 11, input: "a = 1", diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@try.py.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@try.py.snap new file mode 100644 index 00000000..43369a02 --- /dev/null +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__complete@try.py.snap @@ -0,0 +1,430 @@ +--- +source: parser/src/parser/parser.rs +description: "# in some cases last price or adj price is undefined\ntry:\n last_price = int(price_section[2])\n# when instead of number value is `F`\nexcept (ValueError, IndexError):\n last_price = None\ntry:\n adj_close = int(price_section[3])\nexcept (ValueError, IndexError):\n adj_close = None\ntry:\n market_cap = adj_close * self.total_shares\nexcept ValueError:\n market_cap = None\n\n" +input_file: parser/test_data/inputs/try.py +--- +Module { + node: Node { + start: 0, + end: 381, + }, + body: [ + TryStatement( + Try { + node: Node { + start: 53, + end: 190, + }, + body: [ + AssignStatement( + Assign { + node: Node { + start: 62, + end: 96, + }, + targets: [ + Name( + Name { + node: Node { + start: 62, + end: 72, + }, + id: "last_price", + }, + ), + ], + value: Call( + Call { + node: Node { + start: 75, + end: 96, + }, + func: Name( + Name { + node: Node { + start: 75, + end: 78, + }, + id: "int", + }, + ), + args: [ + Subscript( + Subscript { + node: Node { + start: 79, + end: 95, + }, + value: Name( + Name { + node: Node { + start: 79, + end: 92, + }, + id: "price_section", + }, + ), + slice: Constant( + Constant { + node: Node { + start: 93, + end: 94, + }, + value: Int( + "2", + ), + }, + ), + }, + ), + ], + keywords: [], + starargs: None, + kwargs: None, + }, + ), + }, + ), + ], + handlers: [ + ExceptHandler { + node: Node { + start: 135, + end: 190, + }, + typ: Some( + Tuple( + Tuple { + node: Node { + start: 142, + end: 165, + }, + elements: [ + Name( + Name { + node: Node { + start: 143, + end: 153, + }, + id: "ValueError", + }, + ), + Name( + Name { + node: Node { + start: 155, + end: 165, + }, + id: "IndexError", + }, + ), + ], + }, + ), + ), + name: None, + body: [ + AssignStatement( + Assign { + node: Node { + start: 172, + end: 189, + }, + targets: [ + Name( + Name { + node: Node { + start: 172, + end: 182, + }, + id: "last_price", + }, + ), + ], + value: Constant( + Constant { + node: Node { + start: 185, + end: 189, + }, + value: None, + }, + ), + }, + ), + ], + }, + ], + orelse: [], + finalbody: [], + }, + ), + TryStatement( + Try { + node: Node { + start: 190, + end: 287, + }, + body: [ + AssignStatement( + Assign { + node: Node { + start: 199, + end: 232, + }, + targets: [ + Name( + Name { + node: Node { + start: 199, + end: 208, + }, + id: "adj_close", + }, + ), + ], + value: Call( + Call { + node: Node { + start: 211, + end: 232, + }, + func: Name( + Name { + node: Node { + start: 211, + end: 214, + }, + id: "int", + }, + ), + args: [ + Subscript( + Subscript { + node: Node { + start: 215, + end: 231, + }, + value: Name( + Name { + node: Node { + start: 215, + end: 228, + }, + id: "price_section", + }, + ), + slice: Constant( + Constant { + node: Node { + start: 229, + end: 230, + }, + value: Int( + "3", + ), + }, + ), + }, + ), + ], + keywords: [], + starargs: None, + kwargs: None, + }, + ), + }, + ), + ], + handlers: [ + ExceptHandler { + node: Node { + start: 233, + end: 287, + }, + typ: Some( + Tuple( + Tuple { + node: Node { + start: 240, + end: 263, + }, + elements: [ + Name( + Name { + node: Node { + start: 241, + end: 251, + }, + id: "ValueError", + }, + ), + Name( + Name { + node: Node { + start: 253, + end: 263, + }, + id: "IndexError", + }, + ), + ], + }, + ), + ), + name: None, + body: [ + AssignStatement( + Assign { + node: Node { + start: 270, + end: 286, + }, + targets: [ + Name( + Name { + node: Node { + start: 270, + end: 279, + }, + id: "adj_close", + }, + ), + ], + value: Constant( + Constant { + node: Node { + start: 282, + end: 286, + }, + value: None, + }, + ), + }, + ), + ], + }, + ], + orelse: [], + finalbody: [], + }, + ), + TryStatement( + Try { + node: Node { + start: 287, + end: 381, + }, + body: [ + AssignStatement( + Assign { + node: Node { + start: 296, + end: 338, + }, + targets: [ + Name( + Name { + node: Node { + start: 296, + end: 306, + }, + id: "market_cap", + }, + ), + ], + value: BinOp( + BinOp { + node: Node { + start: 309, + end: 338, + }, + op: Mult, + left: Name( + Name { + node: Node { + start: 309, + end: 318, + }, + id: "adj_close", + }, + ), + right: Attribute( + Attribute { + node: Node { + start: 321, + end: 338, + }, + value: Name( + Name { + node: Node { + start: 321, + end: 325, + }, + id: "self", + }, + ), + attr: "total_shares", + }, + ), + }, + ), + }, + ), + ], + handlers: [ + ExceptHandler { + node: Node { + start: 339, + end: 381, + }, + typ: Some( + Name( + Name { + node: Node { + start: 346, + end: 356, + }, + id: "ValueError", + }, + ), + ), + name: None, + body: [ + AssignStatement( + Assign { + node: Node { + start: 362, + end: 379, + }, + targets: [ + Name( + Name { + node: Node { + start: 362, + end: 372, + }, + id: "market_cap", + }, + ), + ], + value: Constant( + Constant { + node: Node { + start: 375, + end: 379, + }, + value: None, + }, + ), + }, + ), + ], + }, + ], + orelse: [], + finalbody: [], + }, + ), + ], +} diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@from_import.py-2.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@from_import.py-2.snap index 4bc0dc48..e0a564b0 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@from_import.py-2.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@from_import.py-2.snap @@ -5,7 +5,7 @@ input_file: parser/test_data/inputs/one_liners/from_import.py --- [ InvalidSyntax { - path: "", + path: "/Users/glyphack/Programming/enderpy/parser/test_data/inputs/one_liners/from_import.py", msg: "Expected one of \"Identifier, Mul, \" but found Eof", line: 16, input: "", diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@lists.py-7.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@lists.py-7.snap index be4502ac..60934d2c 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@lists.py-7.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@lists.py-7.snap @@ -13,7 +13,7 @@ Module { Assign { node: Node { start: 0, - end: 216, + end: 233, }, targets: [ Name( @@ -30,7 +30,7 @@ Module { List { node: Node { start: 14, - end: 216, + end: 233, }, elements: [ Constant( diff --git a/parser/test_data/inputs/functions.py b/parser/test_data/inputs/functions.py index 487b19a6..d3b7497a 100644 --- a/parser/test_data/inputs/functions.py +++ b/parser/test_data/inputs/functions.py @@ -15,3 +15,17 @@ def _extract_ticker_client_types_data(ticker_index: str) -> List: response = session.get(url, timeout=5) data = response.text.split(';') return data + +def common_process(df: pd.DataFrame, date: str): + if len(df) == 0: + return pd.DataFrame(columns=list(api_to_orderbook_mapping.keys())) + df.rename(columns=reversed_keys, inplace=True) + df = df.loc[:, list(api_to_orderbook_mapping.keys())] + df["datetime"] = pd.to_datetime( + date + " " + df["datetime"].astype(str), format="%Y%m%d %H%M%S" + ) + df = df.sort_values(["datetime", "depth"], ascending=[True, True]) + df.set_index("datetime", inplace=True) + df.drop(columns=["refID"], inplace=True) + return df + diff --git a/parser/test_data/inputs/try.py b/parser/test_data/inputs/try.py new file mode 100644 index 00000000..d6aeb892 --- /dev/null +++ b/parser/test_data/inputs/try.py @@ -0,0 +1,15 @@ +# in some cases last price or adj price is undefined +try: + last_price = int(price_section[2]) +# when instead of number value is `F` +except (ValueError, IndexError): + last_price = None +try: + adj_close = int(price_section[3]) +except (ValueError, IndexError): + adj_close = None +try: + market_cap = adj_close * self.total_shares +except ValueError: + market_cap = None +