From 22f2d8782109dba8895d009b3cef80c772a37f7a Mon Sep 17 00:00:00 2001 From: Glyphack Date: Thu, 10 Oct 2024 20:39:05 +0200 Subject: [PATCH] Remove constant values from ast tree --- enderpy/src/main.rs | 1 - parser/src/ast.rs | 51 +++++++----------- parser/src/parser/compat.rs | 24 +-------- parser/src/parser/mod.rs | 25 +++++---- parser/src/parser/parser.rs | 72 ++++++++++--------------- typechecker/src/type_evaluator.rs | 87 ++++++++++++++++--------------- typechecker/src/types.rs | 28 +++------- 7 files changed, 110 insertions(+), 178 deletions(-) diff --git a/enderpy/src/main.rs b/enderpy/src/main.rs index ab73022b..acc831ad 100644 --- a/enderpy/src/main.rs +++ b/enderpy/src/main.rs @@ -106,7 +106,6 @@ fn tokenize() -> Result<()> { fn parse(file: &PathBuf) -> Result<()> { let source = fs::read_to_string(file).into_diagnostic()?; - let file_path = file.to_str().unwrap_or(""); let mut parser = Parser::new(&source); let ast = parser.parse(); println!("{:#?}", ast); diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 6ea687cc..0b792b0e 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -13,6 +13,12 @@ pub struct Node { pub end: u32, } +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] // #[serde(tag = "type")] +pub struct TextRange { + pub start: u32, + pub end: u32, +} + impl fmt::Display for Node { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "({}, {})", self.start, self.end) @@ -342,45 +348,26 @@ pub struct Constant { pub value: ConstantValue, } -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Debug)] pub enum ConstantValue { None, Ellipsis, - Bool(bool), - Str(String), - Bytes(Vec), - Tuple(Vec), + Bool, + Str, + Bytes, + Tuple, // Numbers are string because we don't care about the value rn. - Int(String), - Float(String), - Complex { real: String, imaginary: String }, + Int, + Float, + Complex, } -impl fmt::Debug for ConstantValue { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ConstantValue::None => write!(f, "None"), - ConstantValue::Ellipsis => write!(f, "..."), - ConstantValue::Bool(b) => write!(f, "{}", b), - ConstantValue::Str(s) => { - if s.starts_with("r\"") || s.starts_with("R\"") { - let mut s = s.chars().skip(1).collect::(); - s = s.chars().take(s.len() - 1).collect::(); - - s = s.chars().skip(1).collect::(); - - return write!(f, "{:?}", s); - } - - write!(f, "\"{}\"", s) - } - ConstantValue::Bytes(b) => write!(f, "{:?}", b), - ConstantValue::Tuple(t) => write!(f, "{:?}", t), - ConstantValue::Int(i) => write!(f, "{}", i), - ConstantValue::Float(fl) => write!(f, "{}", fl), - ConstantValue::Complex { real, imaginary } => write!(f, "{}+{}j", real, imaginary), - } + +impl Constant { + pub fn get_value<'a>(&self, source: &'a str) -> &'a str { + return &source[self.node.start as usize..self.node.end as usize]; } } + #[derive(Debug, Clone)] pub struct List { pub node: Node, diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index 07aaf206..22e90fa4 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -287,33 +287,11 @@ impl AsPythonCompat for Name { impl AsPythonCompat for Constant { fn as_python_compat(&self, parser: &Parser) -> Value { json_python_compat_node!("Constant", self, parser, { - "value": self.value.as_python_compat(parser), + "value": json!(self.get_value(parser.source)), }) } } -impl AsPythonCompat for ConstantValue { - fn as_python_compat(&self, parser: &Parser) -> Value { - match self { - ConstantValue::None => json!(null), - ConstantValue::Ellipsis => json!("..."), - ConstantValue::Bool(v) => json!(v), - ConstantValue::Str(v) => json!(v), - ConstantValue::Bytes(v) => json!(v), - ConstantValue::Tuple(v) => json!(v - .iter() - .map(|cons| cons.as_python_compat(parser)) - .collect::>()), - ConstantValue::Int(v) => Value::Number(Number::from_str(v).unwrap()), - ConstantValue::Float(v) => Value::Number(Number::from_str(v).unwrap()), - ConstantValue::Complex { - real: _real, - imaginary, - } => json!(imaginary), - } - } -} - impl AsPythonCompat for List { fn as_python_compat(&self, parser: &Parser) -> Value { json_python_compat_node!("List", self, parser, { diff --git a/parser/src/parser/mod.rs b/parser/src/parser/mod.rs index 932f23f0..7789fdbe 100644 --- a/parser/src/parser/mod.rs +++ b/parser/src/parser/mod.rs @@ -64,23 +64,22 @@ pub fn concat_string_exprs(lhs: Expression, rhs: Expression) -> Result { + (ConstantValue::Str, ConstantValue::Str) => { Expression::Constant(Box::new(Constant { node, - value: ConstantValue::Str(lhs_val + &rhs_val), + value: ConstantValue::Str, })) } - (ConstantValue::Bytes(mut lhs), ConstantValue::Bytes(rhs)) => { - lhs.append(&mut rhs.clone()); + (ConstantValue::Bytes, ConstantValue::Bytes) => { Expression::Constant(Box::new(Constant { node, - value: ConstantValue::Bytes(lhs), + value: ConstantValue::Bytes, })) } - (ConstantValue::Bytes(_lhs), _) => { + (ConstantValue::Bytes, _) => { panic!("Cannot concat bytes and string"); } - (_, ConstantValue::Bytes(_rhs)) => { + (_, ConstantValue::Bytes) => { panic!("Can only concat bytes with other bytes"); } _ => panic!("Cannot concat string"), @@ -101,13 +100,13 @@ pub fn concat_string_exprs(lhs: Expression, rhs: Expression) -> Result { let mut values = fstring_lhs.values; match const_rhs.value { - ConstantValue::Str(rhs_val) => { + ConstantValue::Str => { values.push(Expression::Constant(Box::new(Constant { node: const_rhs.node, - value: ConstantValue::Str(rhs_val), + value: ConstantValue::Str, }))); } - ConstantValue::Bytes(_) => { + ConstantValue::Bytes => { panic!("Cannot concat string and bytes"); } _ => panic!("Cannot concat string"), @@ -122,11 +121,11 @@ pub fn concat_string_exprs(lhs: Expression, rhs: Expression) -> Result { let const_expr = match const_lhs.value { - ConstantValue::Str(rhs_val) => Expression::Constant(Box::new(Constant { + ConstantValue::Str => Expression::Constant(Box::new(Constant { node: const_lhs.node, - value: ConstantValue::Str(rhs_val), + value: ConstantValue::Str, })), - ConstantValue::Bytes(_) => { + ConstantValue::Bytes => { panic!("Cannot concat string and bytes"); } _ => panic!("Cannot concat string"), diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 36bfa840..75c1cd2f 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -2443,7 +2443,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Int(val), + value: ConstantValue::Int, })) } Kind::None => { @@ -2457,14 +2457,14 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Bool(true), + value: ConstantValue::Bool, })) } Kind::False => { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Bool(false), + value: ConstantValue::Bool, })) } Kind::ImaginaryInteger => { @@ -2472,10 +2472,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Complex { - real: "0".to_string(), - imaginary: val, - }, + value: ConstantValue::Complex, })) } Kind::Bytes => { @@ -2489,7 +2486,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Bytes(bytes_val), + value: ConstantValue::Bytes, })) } Kind::StringLiteral => { @@ -2498,7 +2495,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Str(string_val), + value: ConstantValue::Str, })) } @@ -2510,7 +2507,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Bytes(bytes_val), + value: ConstantValue::Bytes, })) } Kind::FStringStart => { @@ -2526,7 +2523,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Float(val), + value: ConstantValue::Float, })) } Kind::ExponentFloat => { @@ -2534,7 +2531,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Float(val), + value: ConstantValue::Float, })) } Kind::ImaginaryPointFloat => { @@ -2542,10 +2539,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Complex { - real: "0".to_string(), - imaginary: val, - }, + value: ConstantValue::Complex, })) } Kind::ImaginaryExponentFloat => { @@ -2553,10 +2547,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Complex { - real: "0".to_string(), - imaginary: val, - }, + value: ConstantValue::Complex, })) } Kind::Ellipsis => { @@ -2572,7 +2563,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Int(val), + value: ConstantValue::Int, })) } Kind::Hexadecimal => { @@ -2580,7 +2571,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Int(val), + value: ConstantValue::Int, })) } _ => { @@ -2614,7 +2605,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Int(val), + value: ConstantValue::Int, })) } Kind::None => { @@ -2628,14 +2619,14 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Bool(true), + value: ConstantValue::Bool, })) } Kind::False => { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Bool(false), + value: ConstantValue::Bool, })) } Kind::ImaginaryInteger => { @@ -2643,10 +2634,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Complex { - real: "0".to_string(), - imaginary: val, - }, + value: ConstantValue::Complex, })) } Kind::Bytes => { @@ -2660,7 +2648,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Bytes(bytes_val), + value: ConstantValue::Bytes, })) } Kind::StringLiteral => { @@ -2669,7 +2657,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Str(string_val), + value: ConstantValue::Str, })) } @@ -2682,7 +2670,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Bytes(bytes_val), + value: ConstantValue::Bytes, })) } Kind::FStringStart => { @@ -2698,7 +2686,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Float(val), + value: ConstantValue::Float, })) } Kind::ExponentFloat => { @@ -2706,7 +2694,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Float(val), + value: ConstantValue::Float, })) } Kind::ImaginaryPointFloat => { @@ -2714,10 +2702,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Complex { - real: "0".to_string(), - imaginary: val, - }, + value: ConstantValue::Complex, })) } Kind::ImaginaryExponentFloat => { @@ -2725,10 +2710,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Complex { - real: "0".to_string(), - imaginary: val, - }, + value: ConstantValue::Complex, })) } Kind::Ellipsis => { @@ -2744,7 +2726,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Int(val), + value: ConstantValue::Int, })) } Kind::Hexadecimal => { @@ -2752,7 +2734,7 @@ impl<'a> Parser<'a> { self.bump_any(); Expression::Constant(Box::new(Constant { node: self.finish_node(start), - value: ConstantValue::Int(val), + value: ConstantValue::Int, })) } _ => { @@ -3405,7 +3387,7 @@ impl<'a> Parser<'a> { self.bump(Kind::FStringMiddle); Ok(Expression::Constant(Box::new(Constant { node: self.finish_node(node), - value: ConstantValue::Str(str_val), + value: ConstantValue::Str, }))) } Kind::LeftBracket => self.parse_fstring_replacement_field(), diff --git a/typechecker/src/type_evaluator.rs b/typechecker/src/type_evaluator.rs index 11d76cda..e82ef552 100755 --- a/typechecker/src/type_evaluator.rs +++ b/typechecker/src/type_evaluator.rs @@ -102,16 +102,16 @@ impl<'a> TypeEvaluator<'a> { let typ = match &c.value { // Constants are not literals unless they are explicitly // typing.readthedocs.io/en/latest/spec/literal.html#backwards-compatibility - ast::ConstantValue::Int(_) => self.get_builtin_type("int"), - ast::ConstantValue::Float(_) => self.get_builtin_type("float"), - ast::ConstantValue::Str(_) => self.get_builtin_type("str"), - ast::ConstantValue::Bool(_) => self.get_builtin_type("bool"), + ast::ConstantValue::Int => self.get_builtin_type("int"), + ast::ConstantValue::Float => self.get_builtin_type("float"), + ast::ConstantValue::Str => self.get_builtin_type("str"), + ast::ConstantValue::Bool => self.get_builtin_type("bool"), ast::ConstantValue::None => Some(PythonType::None), - ast::ConstantValue::Bytes(_) => self.get_builtin_type("bytes"), + ast::ConstantValue::Bytes => self.get_builtin_type("bytes"), ast::ConstantValue::Ellipsis => Some(PythonType::Any), // TODO: implement - ast::ConstantValue::Tuple(_) => Some(PythonType::Unknown), - ast::ConstantValue::Complex { real, imaginary } => Some(PythonType::Unknown), + ast::ConstantValue::Tuple => Some(PythonType::Unknown), + ast::ConstantValue::Complex => Some(PythonType::Unknown), }; Ok(match typ { Some(t) => t, @@ -145,7 +145,7 @@ impl<'a> TypeEvaluator<'a> { }; let type_name = match first_arg { ast::Expression::Constant(str) => match &str.value { - ast::ConstantValue::Str(s) => s, + ast::ConstantValue::Str => str.get_value(""), _ => panic!("TypeVar first arg must be a string"), }, _ => panic!("TypeVar must be called with at least one arg"), @@ -561,8 +561,8 @@ impl<'a> TypeEvaluator<'a> { // 2. Module is preferred over local scope so we first check module scope and // then local scope. // https://peps.python.org/pep-0563/#backwards-compatibility - ast::ConstantValue::Str(ref str) => { - let mut parser = Parser::new(str); + ast::ConstantValue::Str => { + let mut parser = Parser::new(""); // Wrap the parsing logic inside a `catch_unwind` block let parse_result = catch_unwind(AssertUnwindSafe(|| parser.parse())); @@ -1320,47 +1320,48 @@ impl<'a> TypeEvaluator<'a> { let val = match expr { Expression::Constant(c) => { match c.value.clone() { - ast::ConstantValue::Bool(b) => LiteralValue::Bool(b), - ast::ConstantValue::Int(i) => LiteralValue::Int(i), - ast::ConstantValue::Float(f) => LiteralValue::Float(f), - ast::ConstantValue::Str(s) => LiteralValue::Str(s), - ast::ConstantValue::Bytes(b) => LiteralValue::Bytes(b), + ast::ConstantValue::Bool => LiteralValue::Bool, + ast::ConstantValue::Int => LiteralValue::Int, + ast::ConstantValue::Float => LiteralValue::Float, + ast::ConstantValue::Str => LiteralValue::Str, + ast::ConstantValue::Bytes => LiteralValue::Bytes, ast::ConstantValue::None => LiteralValue::None, // Tuple is illegal if it has parentheses, otherwise it's allowed and the output // a multiValued type Currently even mypy does not support // this, who am I to do it? https://mypy-play.net/?mypy=latest&python=3.10&gist=0df0421d5c85f3b75f65a51cae8616ce - ast::ConstantValue::Tuple(t) => { - if t.len() == 1 { - match t[0].value.clone() { - ast::ConstantValue::Bool(b) => LiteralValue::Bool(b), - ast::ConstantValue::Int(i) => LiteralValue::Int(i), - ast::ConstantValue::Float(f) => LiteralValue::Float(f), - ast::ConstantValue::Str(s) => LiteralValue::Str(s), - ast::ConstantValue::Bytes(b) => LiteralValue::Bytes(b), - ast::ConstantValue::None => LiteralValue::None, - _ => panic!("Tuple type with illegal parameter"), - } - } else { - let literal_values = t - .iter() - .map(|c| match c.value.clone() { - ast::ConstantValue::Bool(b) => LiteralValue::Bool(b), - ast::ConstantValue::Int(i) => LiteralValue::Int(i), - ast::ConstantValue::Float(f) => LiteralValue::Float(f), - ast::ConstantValue::Str(s) => LiteralValue::Str(s), - ast::ConstantValue::Bytes(b) => LiteralValue::Bytes(b), - ast::ConstantValue::None => LiteralValue::None, - _ => panic!("Tuple type with illegal parameter"), - }) - .collect(); - return literal_values; - } + ast::ConstantValue::Tuple => { + LiteralValue::Int + // if t.len() == 1 { + // match t[0].value.clone() { + // ast::ConstantValue::Bool => LiteralValue::Bool, + // ast::ConstantValue::Int => LiteralValue::Int, + // ast::ConstantValue::Float => LiteralValue::Float, + // ast::ConstantValue::Str => LiteralValue::Str, + // ast::ConstantValue::Bytes => LiteralValue::Bytes, + // ast::ConstantValue::None => LiteralValue::None, + // _ => panic!("Tuple type with illegal parameter"), + // } + // } else { + // let literal_values = t + // .iter() + // .map(|c| match c.value { + // ast::ConstantValue::Bool => LiteralValue::Bool, + // ast::ConstantValue::Int => LiteralValue::Int, + // ast::ConstantValue::Float => LiteralValue::Float, + // ast::ConstantValue::Str => LiteralValue::Str, + // ast::ConstantValue::Bytes => LiteralValue::Bytes, + // ast::ConstantValue::None => LiteralValue::None, + // _ => panic!("Tuple type with illegal parameter"), + // }) + // .collect(); + // return literal_values; + // } } // Illegal parameter ast::ConstantValue::Ellipsis => { panic!("Literal type with ellipsis value is not supported") } - ast::ConstantValue::Complex { real, imaginary } => { + ast::ConstantValue::Complex => { panic!("Literal type with complex value is not supported") } } @@ -1371,7 +1372,7 @@ impl<'a> TypeEvaluator<'a> { Expression::Name(n) => &n.id, _ => panic!("Literal type with attribute value can only be a name"), }; - LiteralValue::Str(value.to_string()) + LiteralValue::Str } Expression::Subscript(s) => { match &s.value { diff --git a/typechecker/src/types.rs b/typechecker/src/types.rs index cd106593..39b7697b 100644 --- a/typechecker/src/types.rs +++ b/typechecker/src/types.rs @@ -317,31 +317,17 @@ pub struct KnownValue { #[derive(Debug, Clone, Eq, PartialEq)] pub enum LiteralValue { - Bool(bool), - Int(String), - Float(String), - Str(String), + Bool, + Int, + Float, + Str, None, - Bytes(Vec), + Bytes, } impl Display for LiteralValue { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let value_str = match self { - LiteralValue::Bool(b) => b.to_string(), - LiteralValue::Int(i) => i.to_string(), - LiteralValue::Float(f) => f.to_string(), - LiteralValue::Str(s) => s.to_string(), - LiteralValue::None => "None".to_string(), - LiteralValue::Bytes(b) => { - for byte in b { - write!(f, "{:02x}", byte)?; - } - return Ok(()); - } - }; - - write!(f, "{}", value_str) + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "") } }