Not done with the new evaluator

Author: mxgordon

Cargo.lock

@@ -11,6 +11,7 @@ strum = "0.26.3"
 strum_macros = "0.26.4"
 mopa = "0.2.2"
 ahash = "0.8.11"
+rustpython-parser = "0.4.0"
 
 [profile.release]
 lto = "fat"

Cargo.toml

@@ -1,22 +1,22 @@
+use rustpython_parser::ast::located::CmpOp;
 use crate::builtins::function_utils::call_function_1_arg_min;
 use crate::builtins::structure::magic_methods::PyMagicMethod;
 use crate::builtins::structure::pyobject::{FuncReturnType, PyObject};
-use crate::parser::Comparator;
 use crate::pyarena::PyArena;
 
-pub fn compare_op(left: &PyObject, right: &PyObject, comp: &Comparator, arena: &mut PyArena) -> FuncReturnType {
+pub fn compare_op(left: &PyObject, right: &PyObject, comp: &CmpOp, arena: &mut PyArena) -> FuncReturnType {
  match comp {
- Comparator::Equal => left_hand_compare_op(&PyMagicMethod::Eq, left, right, arena),
- Comparator::NotEqual => left_hand_compare_op(&PyMagicMethod::Ne, left, right, arena),
- Comparator::LessThan => left_hand_compare_op(&PyMagicMethod::Lt, left, right, arena),
- Comparator::LessThanOrEqual => left_hand_compare_op(&PyMagicMethod::Le, left, right, arena),
- Comparator::GreaterThan => left_hand_compare_op(&PyMagicMethod::Gt, left, right, arena),
- Comparator::GreaterThanOrEqual => left_hand_compare_op(&PyMagicMethod::Ge, left, right, arena),
+ CmpOp::Eq => left_hand_compare_op(&PyMagicMethod::Eq, left, right, arena),
+ CmpOp::NotEq => left_hand_compare_op(&PyMagicMethod::Ne, left, right, arena),
+ CmpOp::Lt => left_hand_compare_op(&PyMagicMethod::Lt, left, right, arena),
+ CmpOp::LtE => left_hand_compare_op(&PyMagicMethod::Le, left, right, arena),
+ CmpOp::Gt => left_hand_compare_op(&PyMagicMethod::Gt, left, right, arena),
+ CmpOp::GtE => left_hand_compare_op(&PyMagicMethod::Ge, left, right, arena),
 
- Comparator::Is => Ok(is_compare(false, left, right, arena)),
- Comparator::IsNot => Ok(is_compare(true, left, right, arena)),
- Comparator::In => {todo!()}
- Comparator::NotIn => {todo!()}
+ CmpOp::Is => Ok(is_compare(false, left, right, arena)),
+ CmpOp::IsNot => Ok(is_compare(true, left, right, arena)),
+ CmpOp::In => {todo!()}
+ CmpOp::NotIn => {todo!()}
  }
 }
 

src/builtins/functions/compare.rs

@@ -23,7 +23,7 @@ pub fn math_op(left: PyObject, right: PyObject, py_magic_method: PyMagicMethod,
  right_hand_math_op(left, right, py_magic_method, arena)
 }
 
-fn right_hand_math_op(left: PyObject, right: PyObject, mut py_magic_method: PyMagicMethod,  arena: &mut PyArena) -> FuncReturnType {
+pub(crate) fn right_hand_math_op(left: PyObject, right: PyObject, mut py_magic_method: PyMagicMethod, arena: &mut PyArena) -> FuncReturnType {
  py_magic_method.make_right_handed();
 
  let right_math_func = right.get_magic_method(&py_magic_method, arena);

src/builtins/functions/math_op.rs

@@ -47,14 +47,14 @@ impl Globals {
  }
  }
 
- pub fn create_exposed_globals(&self) -> AHashMap<String, Cell<PyObject>> {
+ pub fn create_exposed_globals(&self) -> AHashMap<String, PyObject> {
  vec![
- ("object".to_string(), Cell::new(PyObject::new_internal_class(self.object_class.clone()))),
- ("int".to_string(), Cell::new(PyObject::new_internal_class(self.int_class.clone()))),
- ("bool".to_string(), Cell::new(PyObject::new_internal_class(self.bool_class.clone()))),
- ("float".to_string(), Cell::new(PyObject::new_internal_class(self.float_class.clone()))),
- ("range".to_string(), Cell::new(PyObject::new_internal_class(self.range_class.clone()))),
- ("print".to_string(), Cell::new(PyObject::new_internal_func(self.print_func.clone()))),
+ ("object".to_string(), PyObject::new_internal_class(self.object_class.clone())),
+ ("int".to_string(), PyObject::new_internal_class(self.int_class.clone())),
+ ("bool".to_string(), PyObject::new_internal_class(self.bool_class.clone())),
+ ("float".to_string(), PyObject::new_internal_class(self.float_class.clone())),
+ ("range".to_string(), PyObject::new_internal_class(self.range_class.clone())),
+ ("print".to_string(), PyObject::new_internal_func(self.print_func.clone())),
  ].into_iter().collect()
  }
 

src/builtins/globals.rs

@@ -1,19 +1,28 @@
 use std::cell::{Ref, RefCell, RefMut};
 use std::fmt::Debug;
 use std::rc::Rc;
-use crate::parser::CodeBlock;
+// use crate::parser::CodeBlock;
 use crate::pyarena::PyArena;
 use crate::builtins::structure::magic_methods::{PyMagicMethod};
 use crate::builtins::structure::pyclass::PyClass;
 use crate::builtins::structure::pyexception::PyException;
 use crate::builtins::structure::pyinstance::PyInstance;
 
+#[derive(Debug)]
+pub enum StatementOperation {
+ Return(Option<PyObject>),
+ Expression(PyObject),
+ Pass,
+ Continue,
+ Break
+}
+
 #[derive(Clone, Debug)]
 pub enum PyObject {
  Immutable(Rc<PyImmutableObject>), // TODO, consider using Cow's here
  Mutable(PyPointer<PyMutableObject>),
  Internal(PyInternalObject),
- IteratorFlag(PyIteratorFlag)
+ // IteratorFlag(PyIteratorFlag)
 }
 
 impl PyObject {
@@ -48,14 +57,14 @@ impl PyObject {
  }
 
 
- pub fn break_() -> Self { // TODO prob should be moved out of the pyobject class (currently in here for legacy reasons)
- PyObject::IteratorFlag(PyIteratorFlag::Break)
- }
- 
- pub fn continue_() -> Self { // TODO prob should be moved out of the pyobject class (currently in here for legacy reasons)
- PyObject::IteratorFlag(PyIteratorFlag::Continue)
- }
- 
+ // pub fn break_() -> Self { // TODO prob should be moved out of the pyobject class (currently in here for legacy reasons)
+ //  PyObject::IteratorFlag(PyIteratorFlag::Break)
+ // }
+ // 
+ // pub fn continue_() -> Self { // TODO prob should be moved out of the pyobject class (currently in here for legacy reasons)
+ //  PyObject::IteratorFlag(PyIteratorFlag::Continue)
+ // }
+ // 
  pub fn create_new_none() -> Self {
  PyObject::Immutable(Rc::new(PyImmutableObject::None))
  }
@@ -68,9 +77,9 @@ impl PyObject {
  Self::new_immutable(PyImmutableObject::Bool(value))
  }
 
- pub fn stop_iteration() -> Self { // TODO prob should be moved out of the pyobject class (currently in here for legacy reasons)
- PyObject::IteratorFlag(PyIteratorFlag::StopIteration)
- }
+ // pub fn stop_iteration() -> Self { // TODO prob should be moved out of the pyobject class (currently in here for legacy reasons)
+ //  PyObject::IteratorFlag(PyIteratorFlag::StopIteration)
+ // }
 
  pub fn expect_immutable(&self) -> &Rc<PyImmutableObject> {
  match self {
@@ -97,7 +106,7 @@ impl PyObject {
  match self {
  PyObject::Immutable(inner) => inner.get_magic_method(py_magic_method, arena),
  PyObject::Mutable(inner) => inner.borrow().get_magic_method(py_magic_method, arena),
- PyObject::IteratorFlag(_) => {panic!("IteratorFlag has no magic methods")}
+ // PyObject::IteratorFlag(_) => {panic!("IteratorFlag has no magic methods")}
  PyObject::Internal(_) => {todo!()}
  }
  }
@@ -106,7 +115,7 @@ impl PyObject {
  match *self {
  PyObject::Immutable(ref inner) => inner.get_class(arena).clone(),
  PyObject::Mutable(ref inner) => inner.borrow().get_class().clone(),
- PyObject::IteratorFlag(_) => {panic!("IteratorFlag has no class")}
+ // PyObject::IteratorFlag(_) => {panic!("IteratorFlag has no class")}
  PyObject::Internal(_) => {todo!()}
  }
  }
@@ -116,7 +125,7 @@ impl PyObject {
  PyObject::Immutable(immutable) => &*immutable.clone() as *const PyImmutableObject as usize,
  PyObject::Mutable(mutable) => &*mutable.clone().borrow() as *const PyMutableObject as usize,
  PyObject::Internal(internal) => internal.get_memory_location(),
- PyObject::IteratorFlag(flag) => flag as *const PyIteratorFlag as usize
+ // PyObject::IteratorFlag(flag) => flag as *const PyIteratorFlag as usize
  }
  }
 }
@@ -189,25 +198,25 @@ impl PyInternalObject {
 
 #[derive(Debug)]
 pub enum PyMutableObject {
- // Class(Rc<PyClass>),
+ Class(Rc<PyClass>),
  Instance(PyInstance),
- Function(PyFunction),
+ // Function(PyFunction),
 }
 
 impl PyMutableObject {
  pub fn get_class(&self) -> &Rc<PyClass> {
  match self {
  PyMutableObject::Instance(py_instance) => py_instance.get_class(),
- // PyMutableObject::Class(py_class) => py_class,
- PyMutableObject::Function(_py_function) => todo!(),
+ PyMutableObject::Class(py_class) => py_class,
+ // PyMutableObject::Function(_py_function) => todo!(),
  }
  }
 
  pub fn get_field(&self, name: &str, arena: &mut PyArena) -> FuncReturnType {
  match self {
  PyMutableObject::Instance(instance) => instance.get_field(name, arena),
- // PyMutableObject::Class(py_class) => todo!(),
- PyMutableObject::Function(_py_function) => todo!(),
+ PyMutableObject::Class(py_class) => todo!(),
+ // PyMutableObject::Function(_py_function) => todo!(),
  }
  }
 
@@ -228,19 +237,19 @@ impl PyMutableObject {
 
  pub fn get_magic_method(&self, py_magic_method: &PyMagicMethod, _arena: &mut PyArena) -> Option<PyObject> {
  match self {
- // PyMutableObject::Class(_) => {todo!()}
+ PyMutableObject::Class(_) => {todo!()}
  PyMutableObject::Instance(instance) => { instance.get_class().search_for_magic_method(py_magic_method) }
- PyMutableObject::Function(_) => {todo!()}
+ // PyMutableObject::Function(_) => {todo!()}
  }
  }
 }
 
-#[derive(Debug)]
-pub struct PyFunction {
- name: String,
- args: Vec<String>,
- body: Vec<CodeBlock>,
-}
+// #[derive(Debug)]
+// pub struct PyFunction {
+//  name: String,
+//  args: Vec<String>,
+//  body: Vec<CodeBlock>,
+// }
 
 pub type FuncReturnType = Result<PyObject, PyException>;
 pub type EmptyFuncReturnType = Result<(), PyException>;

src/builtins/structure/pyobject.rs

@@ -55,7 +55,7 @@ pub fn convert_pyobj_to_bool(pyobj: &PyObject, arena: &mut PyArena) -> Result<bo
  PyObject::Immutable(ref immutable) => convert_immutable_to_bool(immutable, arena),
  PyObject::Mutable(ref mutable) => convert_mutable_to_bool(pyobj, &mutable.borrow(), arena),
  PyObject::Internal(_) => {todo!()}
- PyObject::IteratorFlag(_) => {panic!()}
+ // PyObject::IteratorFlag(_) => {panic!()}
  }
 }
 

src/builtins/types/pybool.rs

@@ -1,26 +1,82 @@
-use std::cell::Ref;
 use std::ops::Deref;
 use std::rc::Rc;
+use rustpython_parser::ast::{Mod, ModModule, Stmt, StmtReturn};
 use crate::builtins::function_utils::{call_function, eval_internal_func, eval_obj_init};
 use crate::builtins::functions::compare::compare_op;
 use crate::builtins::functions::math_op::math_op;
 use crate::builtins::structure::magic_methods::PyMagicMethod;
 use crate::builtins::structure::magic_methods::PyMagicMethod::{Add, Mul, Pow, Sub, TrueDiv};
 use crate::builtins::structure::pyexception::PyException;
-use crate::builtins::structure::pyobject::{EmptyFuncReturnType, FuncReturnType, PyInternalObject, PyIteratorFlag, PyObject};
+use crate::builtins::structure::pyobject::{EmptyFuncReturnType, FuncReturnType, PyInternalObject, PyIteratorFlag, PyObject, StatementOperation};
 use crate::builtins::types::pybool::{convert_pyobj_to_bool};
 use crate::builtins::types::str::py_repr;
-use crate::parser::*;
+// use crate::parser::*;
 use crate::pyarena::PyArena;
 
-pub fn evaluate(code: CodeBlock) {
- let mut arena = PyArena::new();
+pub fn evaluate_mod(code: Mod) {
+ // let mut arena = PyArena::new();
+ let result = match code {
+ Mod::Module(module ) => {evaluate_module(module)},
+ Mod::Interactive(_) => {todo!()}
+ Mod::Expression(_) => {todo!()}
+ Mod::FunctionType(_) => {todo!()}
+ };
 
- let code_result = eval_code_block(&code, &mut arena);
+ // let code_result = eval_code_block(&code, &mut arena);
 
- if let Err(err) = code_result {
- println!("{}", err);
+ print!("Exit Value: {:?}", result);
+ 
+ // if let Err(err) = code_result {
+ // println!("{}", err);
+ // }
+}
+
+fn evaluate_module(module: ModModule) -> Result<(PyArena, i32), PyException> {
+ let mut arena = PyArena::new();
+
+ for statement in module.body {
+ let statement_value = match statement {
+ Stmt::Break(..) => {StatementOperation::Break}
+ Stmt::Continue(..) => {StatementOperation::Continue}
+ Stmt::Pass(..) => {StatementOperation::Pass}
+ Stmt::Return(stmt_return) => {StatementOperation::Return(eval_return_stmt(stmt_return, &mut arena)?)}
+ Stmt::Expr(_) => {}
+ Stmt::While(_) => {}
+ Stmt::If(_) => {}
+ Stmt::For(_) => {}
+ Stmt::Assign(_) => {}
+ Stmt::FunctionDef(_) => {}
+ // not implemented
+ Stmt::AsyncFunctionDef(_) => {todo!()}
+ Stmt::ClassDef(_) => {todo!()}
+ Stmt::Delete(_) => {todo!()}
+ Stmt::TypeAlias(_) => {todo!()}
+ Stmt::AugAssign(_) => {todo!()}
+ Stmt::AnnAssign(_) => {todo!()}
+ Stmt::AsyncFor(_) => {todo!()}
+ Stmt::With(_) => {todo!()}
+ Stmt::AsyncWith(_) => {todo!()}
+ Stmt::Match(_) => {todo!()}
+ Stmt::Raise(_) => {todo!()}
+ Stmt::Try(_) => {todo!()}
+ Stmt::TryStar(_) => {todo!()}
+ Stmt::Assert(_) => {todo!()}
+ Stmt::Import(_) => {todo!()}
+ Stmt::ImportFrom(_) => {todo!()}
+ Stmt::Global(_) => {todo!()}
+ Stmt::Nonlocal(_) => {todo!()}
+ }
+ }
+
+ (arena, 0)
+}
+
+fn eval_return_stmt(stmt: StmtReturn, arena: &mut PyArena) -> Result<Option<PyObject>, PyException> {
+ // for expr in stmt.
+ if let Some(expr) = stmt.value {
+ return eval_expr(*expr, arena);
  }
+ Ok(None)
 }
 
 fn eval_var<'a>(name: &str, arena: &'a PyArena) -> Result<&'a PyObject, PyException> {
@@ -167,7 +223,8 @@ fn eval_expr(expr: &Expr, arena: &mut PyArena) -> FuncReturnType {
 
 fn eval_defn_var(variable: &Rc<Variable>, expr: &Expr, arena: &mut PyArena) -> EmptyFuncReturnType {
  let result = eval_expr(expr, arena)?;
- arena.get_current_frame_mut().set(variable, result);
+ // arena.get_current_frame_mut().set(variable, result);
+ arena.set(variable, result);
 
  Ok(())
 }

src/evaluator.rs

@@ -3,17 +3,16 @@
 #![feature(hash_raw_entry)]
 #![feature(get_mut_unchecked)]
 
-mod parser;
 mod evaluator;
 mod pyarena;
 mod builtins;
-mod preprocessor;
+mod new_evaluator;
 
 use std::env;
 use std::fs::File;
 use std::io::Read;
 use crate::evaluator::{evaluate};
-use crate::parser::{parse_code, remove_comments};
+use rustpython_parser::{lexer::{lex}, parse_tokens, Mode};
 
 #[macro_use]
 extern crate mopa;
@@ -31,23 +30,29 @@ fn main() {
 
  } else if args.len() > 2 {
  panic!("Expect 1 arg for the test file name, got: {}", args.len() - 1);
- } else {
- let filename = &args[1];
- let mut file = File::open("tests/".to_string() + filename).unwrap_or_else(|_| panic!("file not found: {}", filename));
-
+ } 
+ let filename = &args[1];
+ let file_path = "tests/".to_string() + filename;
+ let mut file = File::open(file_path.clone()).unwrap_or_else(|_| panic!("file not found: {}", filename));
+ 
+ {
  let _ = file.read_to_string(&mut contents);
  }
- let contents = remove_comments(&contents);
- let contents = contents.trim();
+ // let contents = remove_comments(&contents);
+ // let contents = contents.trim();
+ 
+ let tokens = lex(&contents, Mode::Module);
+ let ast = parse_tokens(tokens, Mode::Module, &file_path);
 
- let parse_tree = parse_code(contents);
- if let Ok(parse_tree) = parse_tree.0 {
+ // let parse_tree = parse_code(contents);
+ if let Ok(ast) = ast {
 
- println!("{:?}", parse_tree);
+ println!("{:?}", ast);
 
- evaluate(parse_tree);
+ evaluate(ast);
 
- } else if let Err(parse_tree_err) = parse_tree.0 {
- println!("Char: \"{}\"({})\nError: {:?}", contents.chars().nth(parse_tree_err.location.offset).unwrap_or_default(), contents.bytes().nth(parse_tree_err.location.offset).unwrap_or_default(), parse_tree_err);
+ } else if let Err(parse_error) = ast {
+ // println!("Char: \"{}\"({})\nError: {:?}", contents.chars().nth(parse_tree_err.location.offset).unwrap_or_default(), contents.bytes().nth(parse_tree_err.location.offset).unwrap_or_default(), parse_tree_err);
+ println!("{:?}", parse_error);
  }
 }