fix: dont remove leading whitespace from non-docstring ending lines

Author: ryangjchandler

src/lexer/mod.rs

@@ -804,16 +804,20 @@ impl Lexer {
  }
  }
 
+ let mut leading_whitespace_buffer = Vec::new();
+
  // If the type of whitespace is the same, we want to know
  // how much whitespace is on this line. We only care about
  // the smallest amount of whitespace in this case.
  loop {
  match (current_indentation_type, state.peek_buf()) {
  (DocStringIndentationType::Space, [b' ', ..]) => {
+ leading_whitespace_buffer.push(b' ');
  current_indentation_amount += 1;
  state.next();
  },
  (DocStringIndentationType::Tab, [b'\t', ..]) => {
+ leading_whitespace_buffer.push(b'\t');
  current_indentation_amount += 1;
  state.next();
  },
@@ -863,6 +867,7 @@ impl Lexer {
  state.set(StackFrame::Scripting)?;
  break TokenKind::EndDocString(label, indentation_type, current_indentation_amount);
  } else {
+ buffer.extend(leading_whitespace_buffer);
  buffer.extend(whitespace_buffer);
  continue;
  }

tests/0231/ast.txt

@@ -0,0 +1,7 @@
+[
+ Expression {
+ expr: Nowdoc {
+ value: " Hello, world!\n Hello, world!",
+ },
+ },
+]

tests/0231/code.php

@@ -0,0 +1,6 @@
+<?php
+
+<<<'EOF'
+ Hello, world!
+ Hello, world!
+EOF;

tests/0231/tokens.txt

@@ -0,0 +1,48 @@
+[
+ Token {
+ kind: OpenTag(
+ Full,
+ ),
+ span: (
+ 1,
+ 1,
+ ),
+ },
+ Token {
+ kind: StartDocString(
+ "EOF",
+ Nowdoc,
+ ),
+ span: (
+ 3,
+ 1,
+ ),
+ },
+ Token {
+ kind: StringPart(
+ " Hello, world!\n Hello, world!",
+ ),
+ span: (
+ 4,
+ 1,
+ ),
+ },
+ Token {
+ kind: EndDocString(
+ "EOF",
+ None,
+ 0,
+ ),
+ span: (
+ 4,
+ 1,
+ ),
+ },
+ Token {
+ kind: SemiColon,
+ span: (
+ 6,
+ 4,
+ ),
+ },
+]