Merge #11163

11163: internal: start tests for top-level parser entry points r=matklad a=matklad bors r+ 🤖 Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

Author: bors[bot]

crates/parser/src/tests.rs

@@ -1,5 +1,6 @@
 mod sourcegen_inline_tests;
-mod entries;
+mod prefix_entries;
+mod top_entries;
 
 use std::{
  fmt::Write,
@@ -9,7 +10,7 @@ use std::{
 
 use expect_test::expect_file;
 
-use crate::LexedStr;
+use crate::{LexedStr, TopEntryPoint};
 
 #[test]
 fn lex_ok() {
@@ -45,7 +46,7 @@ fn lex(text: &str) -> String {
 #[test]
 fn parse_ok() {
  for case in TestCase::list("parser/ok") {
- let (actual, errors) = parse(&case.text);
+ let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
  assert!(!errors, "errors in an OK file {}:\n{}", case.rs.display(), actual);
  expect_file![case.txt].assert_eq(&actual);
  }
@@ -54,7 +55,7 @@ fn parse_ok() {
 #[test]
 fn parse_inline_ok() {
  for case in TestCase::list("parser/inline/ok") {
- let (actual, errors) = parse(&case.text);
+ let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
  assert!(!errors, "errors in an OK file {}:\n{}", case.rs.display(), actual);
  expect_file![case.txt].assert_eq(&actual);
  }
@@ -63,7 +64,7 @@ fn parse_inline_ok() {
 #[test]
 fn parse_err() {
  for case in TestCase::list("parser/err") {
- let (actual, errors) = parse(&case.text);
+ let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
  assert!(errors, "no errors in an ERR file {}:\n{}", case.rs.display(), actual);
  expect_file![case.txt].assert_eq(&actual)
  }
@@ -72,36 +73,48 @@ fn parse_err() {
 #[test]
 fn parse_inline_err() {
  for case in TestCase::list("parser/inline/err") {
- let (actual, errors) = parse(&case.text);
+ let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
  assert!(errors, "no errors in an ERR file {}:\n{}", case.rs.display(), actual);
  expect_file![case.txt].assert_eq(&actual)
  }
 }
 
-fn parse(text: &str) -> (String, bool) {
+fn parse(entry: TopEntryPoint, text: &str) -> (String, bool) {
  let lexed = LexedStr::new(text);
  let input = lexed.to_input();
- let output = crate::TopEntryPoint::SourceFile.parse(&input);
+ let output = entry.parse(&input);
 
  let mut buf = String::new();
  let mut errors = Vec::new();
  let mut indent = String::new();
+ let mut depth = 0;
+ let mut len = 0;
  lexed.intersperse_trivia(&output, &mut |step| match step {
  crate::StrStep::Token { kind, text } => {
+ assert!(depth > 0);
+ len += text.len();
  write!(buf, "{}", indent).unwrap();
  write!(buf, "{:?} {:?}\n", kind, text).unwrap();
  }
  crate::StrStep::Enter { kind } => {
+ assert!(depth > 0 || len == 0);
+ depth += 1;
  write!(buf, "{}", indent).unwrap();
  write!(buf, "{:?}\n", kind).unwrap();
  indent.push_str(" ");
  }
  crate::StrStep::Exit => {
+ assert!(depth > 0);
+ depth -= 1;
  indent.pop();
  indent.pop();
  }
- crate::StrStep::Error { msg, pos } => errors.push(format!("error {}: {}\n", pos, msg)),
+ crate::StrStep::Error { msg, pos } => {
+ assert!(depth > 0);
+ errors.push(format!("error {}: {}\n", pos, msg))
+ }
  });
+ assert_eq!(len, text.len());
 
  for (token, msg) in lexed.errors() {
  let pos = lexed.text_start(token);

crates/parser/src/tests/entries.rs

@@ -0,0 +1,107 @@
+use crate::{LexedStr, PrefixEntryPoint, Step};
+
+#[test]
+fn vis() {
+ check(PrefixEntryPoint::Vis, "pub(crate) fn foo() {}", "pub(crate)");
+ check(PrefixEntryPoint::Vis, "fn foo() {}", "");
+ check(PrefixEntryPoint::Vis, "pub(fn foo() {}", "pub");
+ check(PrefixEntryPoint::Vis, "pub(crate fn foo() {}", "pub(crate");
+ check(PrefixEntryPoint::Vis, "crate fn foo() {}", "crate");
+}
+
+#[test]
+fn block() {
+ check(PrefixEntryPoint::Block, "{}, 92", "{}");
+ check(PrefixEntryPoint::Block, "{, 92)", "{, 92)");
+ check(PrefixEntryPoint::Block, "()", "");
+}
+
+#[test]
+fn stmt() {
+ check(PrefixEntryPoint::Stmt, "92; fn", "92");
+ check(PrefixEntryPoint::Stmt, "let _ = 92; 1", "let _ = 92");
+ check(PrefixEntryPoint::Stmt, "pub fn f() {} = 92", "pub fn f() {}");
+ check(PrefixEntryPoint::Stmt, "struct S;;", "struct S;");
+ check(PrefixEntryPoint::Stmt, "fn f() {};", "fn f() {}");
+ check(PrefixEntryPoint::Stmt, ";;;", ";");
+ check(PrefixEntryPoint::Stmt, "+", "+");
+ check(PrefixEntryPoint::Stmt, "@", "@");
+ check(PrefixEntryPoint::Stmt, "loop {} - 1", "loop {}");
+}
+
+#[test]
+fn pat() {
+ check(PrefixEntryPoint::Pat, "x y", "x");
+ check(PrefixEntryPoint::Pat, "fn f() {}", "fn");
+ // FIXME: This one is wrong, we should consume only one pattern.
+ check(PrefixEntryPoint::Pat, ".. ..", ".. ..");
+}
+
+#[test]
+fn ty() {
+ check(PrefixEntryPoint::Ty, "fn() foo", "fn()");
+ check(PrefixEntryPoint::Ty, "Clone + Copy + fn", "Clone + Copy +");
+ check(PrefixEntryPoint::Ty, "struct f", "struct");
+}
+
+#[test]
+fn expr() {
+ check(PrefixEntryPoint::Expr, "92 92", "92");
+ check(PrefixEntryPoint::Expr, "+1", "+");
+ check(PrefixEntryPoint::Expr, "-1", "-1");
+ check(PrefixEntryPoint::Expr, "fn foo() {}", "fn");
+ check(PrefixEntryPoint::Expr, "#[attr] ()", "#[attr] ()");
+}
+
+#[test]
+fn path() {
+ check(PrefixEntryPoint::Path, "foo::bar baz", "foo::bar");
+ check(PrefixEntryPoint::Path, "foo::<> baz", "foo::<>");
+ check(PrefixEntryPoint::Path, "foo<> baz", "foo<>");
+ check(PrefixEntryPoint::Path, "Fn() -> i32?", "Fn() -> i32");
+ // FIXME: This shouldn't be accepted as path actually.
+ check(PrefixEntryPoint::Path, "<_>::foo", "<_>::foo");
+}
+
+#[test]
+fn item() {
+ // FIXME: This shouldn't consume the semicolon.
+ check(PrefixEntryPoint::Item, "fn foo() {};", "fn foo() {};");
+ check(PrefixEntryPoint::Item, "#[attr] pub struct S {} 92", "#[attr] pub struct S {}");
+ check(PrefixEntryPoint::Item, "item!{}?", "item!{}");
+ check(PrefixEntryPoint::Item, "????", "?");
+}
+
+#[test]
+fn meta_item() {
+ check(PrefixEntryPoint::MetaItem, "attr, ", "attr");
+ check(PrefixEntryPoint::MetaItem, "attr(some token {stream});", "attr(some token {stream})");
+ check(PrefixEntryPoint::MetaItem, "path::attr = 2 * 2!", "path::attr = 2 * 2");
+}
+
+#[track_caller]
+fn check(entry: PrefixEntryPoint, input: &str, prefix: &str) {
+ let lexed = LexedStr::new(input);
+ let input = lexed.to_input();
+
+ let mut n_tokens = 0;
+ for step in entry.parse(&input).iter() {
+ match step {
+ Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize,
+ Step::Enter { .. } | Step::Exit | Step::Error { .. } => (),
+ }
+ }
+
+ let mut i = 0;
+ loop {
+ if n_tokens == 0 {
+ break;
+ }
+ if !lexed.kind(i).is_trivia() {
+ n_tokens -= 1;
+ }
+ i += 1;
+ }
+ let buf = &lexed.as_str()[..lexed.text_start(i)];
+ assert_eq!(buf, prefix);
+}