fix: improve nested parsing and cleanup

Author: psteinroe

crates/parser/src/parser.rs

@@ -7,13 +7,14 @@ use crate::syntax_error::SyntaxError;
 use crate::syntax_kind::{SyntaxKind, SyntaxKindType};
 use crate::syntax_node::SyntaxNode;
 
-#[derive(Default, Debug)]
+#[derive(Debug)]
 pub struct Parser {
  inner: GreenNodeBuilder<'static, 'static, SyntaxKind>,
  token_buffer: Vec<(SyntaxKind, String)>,
  curr_depth: i32,
  errors: Vec<SyntaxError>,
  stmts: Vec<RawStmt>,
+ checkpoint: Option<i32>,
  is_parsing_erronous_node: bool,
 }
 
@@ -26,34 +27,64 @@ pub struct Parse {
 
 /// Main parser that controls the cst building process, and collects errors and statements
 impl Parser {
+ pub fn new() -> Self {
+ Self {
+ curr_depth: -1,
+ inner: GreenNodeBuilder::new(),
+ token_buffer: Vec::new(),
+ errors: Vec::new(),
+ stmts: Vec::new(),
+ checkpoint: None,
+ is_parsing_erronous_node: false,
+ }
+ }
+
  pub fn close_until_depth(&mut self, depth: i32) {
- while self.curr_depth >= depth && depth > 0 {
+ while self.curr_depth >= depth {
  self.finish_node();
  self.curr_depth -= 1;
  }
  }
 
+ pub fn set_checkpoint(&mut self, is_parsing_erronous_node: bool) {
+ assert!(self.checkpoint.is_none());
+ assert!(self.token_buffer.is_empty());
+ println!("set_checkpoint at {}", self.curr_depth);
+ self.checkpoint = Some(self.curr_depth);
+ self.is_parsing_erronous_node = is_parsing_erronous_node;
+ }
+
+ pub fn close_checkpoint(&mut self) {
+ self.consume_token_buffer();
+ if self.checkpoint.is_some() {
+ self.close_until_depth(self.checkpoint.unwrap());
+ }
+ self.checkpoint = None;
+ self.is_parsing_erronous_node = false;
+ }
+
+ pub fn start_node(&mut self, kind: SyntaxKind) {
+ self.inner.start_node(kind);
+ }
+
  /// start a new node of `SyntaxKind` at `depth`
  /// handles closing previous nodes if necessary
  /// and consumes token buffer before starting new node
  ///
  /// if `SyntaxKind` is `SyntaxKind::AnyStatement`, sets `is_parsing_erronous_node` to true
- pub fn start_node(&mut self, kind: SyntaxKind, depth: i32) {
+ pub fn start_node_at(&mut self, kind: SyntaxKind, depth: Option<i32>) {
+ let depth = depth.unwrap_or(self.curr_depth + 1);
  // close until target depth
  self.close_until_depth(depth);
 
  self.consume_token_buffer();
 
  self.curr_depth = depth;
- self.inner.start_node(kind);
- if kind == SyntaxKind::AnyStatement {
- self.is_parsing_erronous_node = true;
- }
+ self.start_node(kind);
  }
 
  pub fn finish_node(&mut self) {
  self.inner.finish_node();
- self.is_parsing_erronous_node = false;
  }
 
  /// Drains the token buffer and applies all tokens

crates/parser/src/source_file.rs

@@ -26,7 +26,7 @@ impl Parser {
  pub fn parse_source_file(&mut self, text: &str) {
  let mut lexer = SourceFileToken::lexer(text);
 
- self.start_node(SyntaxKind::SourceFile, 0);
+ self.start_node_at(SyntaxKind::SourceFile, Some(0));
  while let Some(token) = lexer.next() {
  match token {
  Ok(token) => {
@@ -92,7 +92,7 @@ select 1;
 
 ";
 
- let mut parser = Parser::default();
+ let mut parser = Parser::new();
  println!("input {:?}", input);
  parser.parse_source_file(input);
  let parsed = parser.finish();

crates/parser/src/statement.rs

@@ -103,7 +103,6 @@ impl StatementToken {
 
 impl Parser {
  pub fn parse_statement(&mut self, text: &str, at_offset: Option<u32>) {
- println!("#### parse_statement: {}", text);
  let offset = at_offset.unwrap_or(0);
  let range = TextRange::new(
  TextSize::from(offset),
@@ -117,7 +116,6 @@ impl Parser {
  Vec::new().into_iter().peekable()
  }
  };
- println!("pg_query_tokens: {:?}", pg_query_tokens);
 
  let parsed = pg_query::parse(text);
  let proto;
@@ -139,18 +137,19 @@ impl Parser {
  Vec::new().into_iter().peekable()
  }
  };
- println!("pg_query_nodes: {:?}", pg_query_nodes);
 
  let mut lexer = StatementToken::lexer(&text);
 
  // parse root node if no syntax errors
  if pg_query_nodes.peek().is_some() {
  let (node, depth, _) = pg_query_nodes.next().unwrap();
  self.stmt(node.to_enum(), range);
- self.start_node(SyntaxKind::from_pg_query_node(&node), depth);
+ self.start_node_at(SyntaxKind::from_pg_query_node(&node), Some(depth));
+ self.set_checkpoint(false);
  } else {
  // fallback to generic node as root
- self.start_node(SyntaxKind::AnyStatement, 1);
+ self.start_node_at(SyntaxKind::Stmt, None);
+ self.set_checkpoint(true);
  }
 
  while let Some(token) = lexer.next() {
@@ -166,7 +165,7 @@ impl Parser {
  } else {
  // node is within span
  let (node, depth, _) = pg_query_nodes.next().unwrap();
- self.start_node(SyntaxKind::from_pg_query_node(&node), depth);
+ self.start_node_at(SyntaxKind::from_pg_query_node(&node), Some(depth));
  }
  }
 
@@ -192,8 +191,7 @@ impl Parser {
  }
 
  // close up nodes
- self.consume_token_buffer();
- self.close_until_depth(1);
+ self.close_checkpoint();
  }
 }
 
@@ -251,7 +249,7 @@ mod tests {
  fn test_statement_parser() {
  let input = "select *,some_col from contact where id = '123 4 5';";
 
- let mut parser = Parser::default();
+ let mut parser = Parser::new();
  parser.parse_statement(input, None);
  let parsed = parser.finish();
 
@@ -264,7 +262,7 @@ mod tests {
  fn test_invalid_statement() {
  let input = "select select;";
 
- let mut parser = Parser::default();
+ let mut parser = Parser::new();
  parser.parse_statement(input, None);
  let parsed = parser.finish();
 

crates/parser/src/syntax_kind.rs

@@ -12,8 +12,8 @@ pub enum SyntaxKind {
  Whitespace,
  Newline,
  Tab,
- Word,  // common value for all keywords (select, from, ...)
- AnyStatement, // node for unknown statements (e.g. when parsing fails)
+ Word, // common value for all keywords (select, from, ...)
+ Stmt, // node for unknown statements (e.g. when parsing fails)
  // from here copyied from NodeEnum
  Alias,
  RangeVar,