parser: use separate tokens for heredocs/nowdocs

Author: ryangjchandler

src/parser/ast.rs

@@ -663,6 +663,12 @@ pub enum Expression {
  InterpolatedString {
  parts: Vec<StringPart>,
  },
+ Heredoc {
+ parts: Vec<StringPart>,
+ },
+ Nowdoc {
+ value: ByteString,
+ },
  PropertyFetch {
  target: Box<Self>,
  property: Box<Self>,

src/parser/mod.rs

@@ -13,6 +13,7 @@ use crate::parser::error::ParseResult;
 use crate::parser::internal::ident::is_reserved_ident;
 use crate::parser::internal::precedence::{Associativity, Precedence};
 use crate::parser::state::State;
+use crate::prelude::ByteString;
 use crate::prelude::DefaultMatchArm;
 
 pub mod ast;
@@ -1052,9 +1053,11 @@ impl Parser {
  e
  }
  TokenKind::StringPart(_) => self.interpolated_string(state)?,
- TokenKind::StartHeredoc(_, kind) => {
+ TokenKind::StartHeredoc(label, kind) => {
  let kind = kind.clone();
- self.doc_string(state, kind)?
+ let label = label.clone();
+
+ self.doc_string(state, label, kind)?
  },
  TokenKind::True => {
  let e = Expression::Bool { value: true };
@@ -1568,44 +1571,57 @@ impl Parser {
  Ok(Expression::InterpolatedString { parts })
  }
 
- fn doc_string(&self, state: &mut State, kind: DocStringKind) -> ParseResult<Expression> {
+ fn doc_string(&self, state: &mut State, label: ByteString, kind: DocStringKind) -> ParseResult<Expression> {
  state.next();
 
- let mut parts = Vec::new();
+ Ok(match kind {
+ DocStringKind::Heredoc => {
+ let mut parts = Vec::new();
 
- while !matches!(state.current.kind, TokenKind::EndHeredoc(_, _, _)) {
- if let Some(part) = self.interpolated_string_part(state)? {
- parts.push(part);
- }
- }
+  while !matches!(state.current.kind, TokenKind::EndHeredoc(_, _, _)) {
+  if let Some(part) = self.interpolated_string_part(state)? {
+  parts.push(part);
+  }
+  }
 
- let (indentation_type, indentation_amount) = match state.current.kind {
- TokenKind::EndHeredoc(_, indentation_type, indentation_amount) => (indentation_type, indentation_amount),
- _ => unreachable!(),
- };
- 
- state.next();
+  let (indentation_type, indentation_amount) = match state.current.kind {
+  TokenKind::EndHeredoc(_, indentation_type, indentation_amount) => (indentation_type, indentation_amount),
+  _ => unreachable!(),
+  };
+  
+  state.next();
 
- // FIXME: Can we move this logic above into the loop, by peeking ahead in
- // the token stream for the EndHeredoc? Might be more performant.
- if let Some(indentation_type) = indentation_type {
- let search_char: u8 = indentation_type.into();
-
- for part in parts.iter_mut() {
- match part {
- StringPart::Const(bytes) => {
- for _ in 0..indentation_amount {
- if bytes.starts_with(&[search_char]) {
- bytes.remove(0);
- }
+ // FIXME: Can we move this logic above into the loop, by peeking ahead in
+ // the token stream for the EndHeredoc? Might be more performant.
+ if let Some(indentation_type) = indentation_type {
+ let search_char: u8 = indentation_type.into();
+
+ for part in parts.iter_mut() {
+ match part {
+ StringPart::Const(bytes) => {
+ for _ in 0..indentation_amount {
+ if bytes.starts_with(&[search_char]) {
+ bytes.remove(0);
+ }
+ }
+ },
+ _ => continue,
  }
- },
- _ => continue,
+ }
  }
- }
- }
 
- Ok(Expression::InterpolatedString { parts })
+ Expression::Heredoc { parts }
+ },
+ DocStringKind::Nowdoc => {
+ // FIXME: This feels hacky. We should probably produce different tokens from the lexer
+ // but since I already had the logic in place for parsing heredocs, this was
+ // the fastest way to get nowdocs working too.
+ let s = expect_token!([TokenKind::StringPart(s) => s], state, "constant string");
+ expect_token!([TokenKind::EndHeredoc(..)], state, "label");
+
+ Expression::Nowdoc { value: s }
+ },
+ })
  }
 
  fn interpolated_string_part(&self, state: &mut State) -> ParseResult<Option<StringPart>> {

tests/0224/ast.txt

@@ -1,6 +1,6 @@
 [
  Expression {
- expr: InterpolatedString {
+ expr: Heredoc {
  parts: [
  Const(
  "Hello, world!",

tests/0224/tokens.txt

@@ -11,6 +11,7 @@
  Token {
  kind: StartHeredoc(
  "EOF",
+ Heredoc,
  ),
  span: (
  3,

tests/0225/ast.txt

@@ -1,6 +1,6 @@
 [
  Expression {
- expr: InterpolatedString {
+ expr: Heredoc {
  parts: [
  Const(
  "Hello, world!",

tests/0225/tokens.txt

@@ -11,6 +11,7 @@
  Token {
  kind: StartHeredoc(
  "TXT",
+ Heredoc,
  ),
  span: (
  3,

tests/0228/ast.txt

@@ -1,6 +1,6 @@
 [
  Expression {
- expr: InterpolatedString {
+ expr: Heredoc {
  parts: [
  Const(
  "Hello, world!\n",

tests/0228/tokens.txt

@@ -11,6 +11,7 @@
  Token {
  kind: StartHeredoc(
  "EOF",
+ Heredoc,
  ),
  span: (
  3,

tests/0229/ast.txt

@@ -1,6 +1,6 @@
 [
  Expression {
- expr: InterpolatedString {
+ expr: Heredoc {
  parts: [
  Const(
  "Hello, world!",

tests/0229/tokens.txt

@@ -11,6 +11,7 @@
  Token {
  kind: StartHeredoc(
  "EOF",
+ Heredoc,
  ),
  span: (
  3,