Preserve whitespace formatting of text nodes

Author: JanTvrdik

src/Lexer/Lexer.php

@@ -139,10 +139,10 @@ private function initialize(): void
 self::TOKEN_EQUAL => '=',
 self::TOKEN_COLON => ':',
 
-self::TOKEN_OPEN_PHPDOC => '/\\*\\*(?=\\s)',
+self::TOKEN_OPEN_PHPDOC => '/\\*\\*(?=\\s)\\x20?+',
 self::TOKEN_CLOSE_PHPDOC => '\\*/',
 self::TOKEN_PHPDOC_TAG => '@[a-z][a-z0-9-]*+',
-self::TOKEN_PHPDOC_EOL => '\\r?+\\n[\\x09\\x20]*+(?:\\*(?!/))?',
+self::TOKEN_PHPDOC_EOL => '\\r?+\\n[\\x09\\x20]*+(?:\\*(?!/)\\x20?+)?',
 
 self::TOKEN_FLOAT => '(?:-?[0-9]++\\.[0-9]*+(?:e-?[0-9]++)?)|(?:-?[0-9]*+\\.[0-9]++(?:e-?[0-9]++)?)|(?:-?[0-9]++e-?[0-9]++)',
 self::TOKEN_INTEGER => '-?(?:(?:0b[0-1]++)|(?:0o[0-7]++)|(?:0x[0-9a-f]++)|(?:[0-9]++))',

src/Parser/PhpDocParser.php

@@ -61,39 +61,27 @@ private function parseChild(TokenIterator $tokens): Ast\PhpDoc\PhpDocChildNode
 private function parseText(TokenIterator $tokens): Ast\PhpDoc\PhpDocTextNode
 {
 $text = '';
-while (true) {
-// If we received a Lexer::TOKEN_PHPDOC_EOL, exit early to prevent
-// them from being processed.
-$currentTokenType = $tokens->currentTokenType();
-if ($currentTokenType === Lexer::TOKEN_PHPDOC_EOL) {
-break;
-}
-$text .= $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END);
-$text = rtrim($text, " \t");
 
-// If we joined until TOKEN_PHPDOC_EOL, peak at the next tokens to see
-// if we have a multiline string to join.
-$currentTokenType = $tokens->currentTokenType();
-if ($currentTokenType !== Lexer::TOKEN_PHPDOC_EOL) {
+while (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
+$text .= $tokens->getSkippedHorizontalWhiteSpaceIfAny() . $tokens->joinUntil(Lexer::TOKEN_PHPDOC_EOL, Lexer::TOKEN_CLOSE_PHPDOC, Lexer::TOKEN_END);
+
+if (!$tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
 break;
 }
 
-// Peek at the next token to determine if it is more text that needs
-// to be combined.
 $tokens->pushSavePoint();
 $tokens->next();
-$currentTokenType = $tokens->currentTokenType();
-if ($currentTokenType !== Lexer::TOKEN_IDENTIFIER) {
+
+if ($tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_TAG) || $tokens->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL) || $tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_PHPDOC) || $tokens->isCurrentTokenType(Lexer::TOKEN_END)) {
 $tokens->rollback();
 break;
 }
 
-// There's more text on a new line, ensure spacing.
+$tokens->dropSavePoint();
 $text .= "\n";
 }
-$text = trim($text, " \t");
 
-return new Ast\PhpDoc\PhpDocTextNode($text);
+return new Ast\PhpDoc\PhpDocTextNode(trim($text, " \t"));
 }
 
 

tests/PHPStan/Parser/PhpDocParserTest.php

@@ -1242,9 +1242,9 @@ public function provideDeprecatedTagsData(): \Iterator
 new PhpDocTagNode(
 '@deprecated',
 new DeprecatedTagValueNode('in Drupal 8.6.0 and will be removed before Drupal 9.0.0. In
-Drupal 9 there will be no way to set the status and in Drupal 8 this
-ability has been removed because mb_*() functions are supplied using
-Symfony\'s polyfill.')
+ Drupal 9 there will be no way to set the status and in Drupal 8 this
+ ability has been removed because mb_*() functions are supplied using
+ Symfony\'s polyfill.')
 ),
 ]),
 ];
@@ -1271,9 +1271,9 @@ public function provideDeprecatedTagsData(): \Iterator
 new PhpDocTagNode(
 '@deprecated',
 new DeprecatedTagValueNode('in Drupal 8.6.0 and will be removed before Drupal 9.0.0. In
-Drupal 9 there will be no way to set the status and in Drupal 8 this
-ability has been removed because mb_*() functions are supplied using
-Symfony\'s polyfill.')
+ Drupal 9 there will be no way to set the status and in Drupal 8 this
+ ability has been removed because mb_*() functions are supplied using
+ Symfony\'s polyfill.')
 ),
 ]),
 ];
@@ -3122,29 +3122,29 @@ public function provideRealWorldExampleData(): \Iterator
 ''
 )
 ),
-new PhpDocTextNode('An empty array if there is no schema, or an associative array with the
-following key/value pairs:'),
-new PhpDocTextNode('- columns: An array of Schema API column specifications, keyed by column
-name. The columns need to be a subset of the properties defined in
-propertyDefinitions(). The \'not null\' property is ignored if present,
-as it is determined automatically by the storage controller depending
-on the table layout and the property definitions. It is recommended to
-avoid having the column definitions depend on field settings when
-possible. No assumptions should be made on how storage engines
-internally use the original column name to structure their storage.'),
-new PhpDocTextNode('- unique keys: (optional) An array of Schema API unique key definitions.
-Only columns that appear in the \'columns\' array are allowed.'),
-new PhpDocTextNode('- indexes: (optional) An array of Schema API index definitions. Only
-columns that appear in the \'columns\' array are allowed. Those indexes
-will be used as default indexes. Field definitions can specify
-additional indexes or, at their own risk, modify the default indexes
-specified by the field-type module. Some storage engines might not
-support indexes.'),
-new PhpDocTextNode('- foreign keys: (optional) An array of Schema API foreign key
-definitions. Note, however, that the field data is not necessarily
-stored in SQL. Also, the possible usage is limited, as you cannot
-specify another field as related, only existing SQL tables,
-such as {taxonomy_term_data}.'),
+new PhpDocTextNode("An empty array if there is no schema, or an associative array with the
+ following key/value pairs:
+ - columns: An array of Schema API column specifications, keyed by column
+ name. The columns need to be a subset of the properties defined in
+ propertyDefinitions(). The 'not null' property is ignored if present,
+ as it is determined automatically by the storage controller depending
+ on the table layout and the property definitions. It is recommended to
+ avoid having the column definitions depend on field settings when
+ possible. No assumptions should be made on how storage engines
+ internally use the original column name to structure their storage.
+ - unique keys: (optional) An array of Schema API unique key definitions.
+ Only columns that appear in the 'columns' array are allowed.
+ - indexes: (optional) An array of Schema API index definitions. Only
+ columns that appear in the 'columns' array are allowed. Those indexes
+ will be used as default indexes. Field definitions can specify
+ additional indexes or, at their own risk, modify the default indexes
+ specified by the field-type module. Some storage engines might not
+ support indexes.
+ - foreign keys: (optional) An array of Schema API foreign key
+ definitions. Note, however, that the field data is not necessarily
+ stored in SQL. Also, the possible usage is limited, as you cannot
+ specify another field as related, only existing SQL tables,
+ such as {taxonomy_term_data}."),
 ]),
 ];
 
@@ -3172,14 +3172,14 @@ public function provideRealWorldExampleData(): \Iterator
 new PhpDocTextNode('Parses a chunked request and return relevant information.'),
 new PhpDocTextNode(''),
 new PhpDocTextNode('This function must return an array containing the following
-keys and their corresponding values:'),
-new PhpDocTextNode('- last: Wheter this is the last chunk of the uploaded file'),
-new PhpDocTextNode('- uuid: A unique id which distinguishes two uploaded files
-This uuid must stay the same among the task of
-uploading a chunked file.'),
-new PhpDocTextNode('- index: A numerical representation of the currently uploaded
-chunk. Must be higher that in the previous request.'),
-new PhpDocTextNode('- orig: The original file name.'),
+ keys and their corresponding values:
+ - last: Wheter this is the last chunk of the uploaded file
+ - uuid: A unique id which distinguishes two uploaded files
+ This uuid must stay the same among the task of
+ uploading a chunked file.
+ - index: A numerical representation of the currently uploaded
+ chunk. Must be higher that in the previous request.
+ - orig: The original file name.'),
 new PhpDocTextNode(''),
 new PhpDocTagNode(
 '@param',
@@ -3201,6 +3201,30 @@ public function provideRealWorldExampleData(): \Iterator
 ]),
 ];
 
+yield [
+'Description with indented <code>',
+"/**
+ * Finder allows searching through directory trees using iterator.
+ *
+ * <code>
+ * Finder::findFiles('*.php')
+ * ->size('> 10kB')
+ * ->from('.')
+ * ->exclude('temp');
+ * </code>
+ */",
+new PhpDocNode([
+new PhpDocTextNode('Finder allows searching through directory trees using iterator.'),
+new PhpDocTextNode(''),
+new PhpDocTextNode("<code>
+Finder::findFiles('*.php')
+ ->size('> 10kB')
+ ->from('.')
+ ->exclude('temp');
+</code>"),
+]),
+];
+
 yield [
 'string literals in @return',
 "/** @return 'foo'|'bar' */",