Merge pull request #52 from supabase/chore/polish

Author: psteinroe

README.md

@@ -33,10 +33,8 @@ Once the parser is stable, and a robust and scalable data model is implemented,
 This is a proof of concept for building both a concrete syntax tree and an abstract syntax tree from a potentially malformed PostgreSQL source code. The `postgres_lsp` crate was created to prove that it works end-to-end, and is just a very basic language server with semantic highlighting and error diagnostics. Before further feature development, we have to complete a bit of groundwork:
 
 1. _Finish the parser_
- - The parser works, but the enum values for all the different syntax elements and internal conversations are manually written or copied, and, in some places, only cover a few elements required for a simple select statement. To have full coverage without possibilities for a copy and paste error, they should be generated from `pg_query.rs` source code. ([#4](https://github.com/supabase/postgres_lsp/pull/4)) ✅
- - There are a few cases such as nested and named dollar quoted strings that cause the parser to fail due to limitations of the regex-based lexer. Nothing that is impossible to fix, or requires any fundamental change in the parser though. ✅
- - Add support for parsing function bodies ([#34](https://github.com/supabase/postgres_lsp/issues/34))
- - Implement a custom parser to extract statements from a source and replace the [regexp-based approach](https://github.com/supabase/postgres_lsp/blob/2f06c49f914b331cf93ea52d9edde8722d27ad05/crates/parser/src/source_parser.rs#L78)
+ - ✅ The core parser algorithm is implemented.
+ - 🚧 But we need help from the community to finalize it: https://github.com/supabase/postgres_lsp/issues/51.
 2. _Implement a robust and scalable data model_
  - This is still in a research phase
  - A great rationale on the importance of the data model in a language server can be found [here](https://matklad.github.io/2023/05/06/zig-language-server-and-cancellation.html)

crates/codegen/src/get_node_properties.rs

@@ -7,12 +7,18 @@ pub fn get_node_properties_mod(proto_file: &ProtoFile) -> proc_macro2::TokenStre
  let node_handlers = node_handlers(&proto_file.nodes);
 
  quote! {
- #[derive(Debug, Clone)]
+ #[derive(Debug, Clone, PartialEq)]
  pub struct TokenProperty {
  pub value: Option<String>,
  pub kind: Option<SyntaxKind>,
  }
 
+ impl TokenProperty {
+ pub fn new(value: Option<String>, kind: Option<SyntaxKind>) -> TokenProperty {
+ TokenProperty { value, kind }
+ }
+ }
+
  impl From<i32> for TokenProperty {
  fn from(value: i32) -> TokenProperty {
  TokenProperty {
@@ -100,6 +106,15 @@ pub fn get_node_properties_mod(proto_file: &ProtoFile) -> proc_macro2::TokenStre
  }
  }
 
+ impl From<SyntaxKind> for TokenProperty {
+ fn from(kind: SyntaxKind) -> TokenProperty {
+ TokenProperty {
+ value: None,
+ kind: Some(kind),
+ }
+ }
+ }
+
  impl From<Token> for TokenProperty {
  fn from(token: Token) -> TokenProperty {
  TokenProperty {

crates/parser/src/codegen.rs

@@ -4,10 +4,18 @@ parser_codegen!();
 
 #[cfg(test)]
 mod tests {
- use crate::codegen::get_nodes;
+ use log::debug;
+
+ use crate::codegen::{get_nodes, SyntaxKind, TokenProperty};
+
+ fn init() {
+ let _ = env_logger::builder().is_test(true).try_init();
+ }
 
  #[test]
  fn test_get_nodes() {
+ init();
+
  let input = "with c as (insert into contact (id) values ('id')) select * from c;";
 
  let pg_query_root = match pg_query::parse(input) {
@@ -27,4 +35,56 @@ mod tests {
  let node_graph = get_nodes(&pg_query_root.unwrap(), 0);
  assert_eq!(node_graph.node_count(), 13);
  }
+
+ fn test_get_node_properties(input: &str, kind: SyntaxKind, expected: Vec<TokenProperty>) {
+ init();
+
+ let pg_query_root = match pg_query::parse(input) {
+ Ok(parsed) => Some(
+ parsed
+ .protobuf
+ .nodes()
+ .iter()
+ .find(|n| n.1 == 1)
+ .unwrap()
+ .0
+ .to_enum(),
+ ),
+ Err(_) => None,
+ };
+
+ let node_graph = get_nodes(&pg_query_root.unwrap(), 0);
+
+ debug!("node graph: {:#?}", node_graph);
+
+ let node_index = node_graph
+ .node_indices()
+ .find(|n| node_graph[*n].kind == kind)
+ .unwrap();
+
+ debug!("selected node: {:#?}", node_graph[node_index]);
+
+ assert_eq!(node_graph[node_index].properties, expected);
+ }
+
+ #[test]
+ fn test_simple_select() {
+ test_get_node_properties(
+ "select 1;",
+ SyntaxKind::SelectStmt,
+ vec![TokenProperty::from(SyntaxKind::Select)],
+ )
+ }
+
+ #[test]
+ fn test_select_with_from() {
+ test_get_node_properties(
+ "select 1 from contact;",
+ SyntaxKind::SelectStmt,
+ vec![
+ TokenProperty::from(SyntaxKind::Select),
+ TokenProperty::from(SyntaxKind::From),
+ ],
+ )
+ }
 }