Fixed T_FN tokenizing when using yield (ref #2706, #2523)

Author: gsherwood

src/Tokenizers/PHP.php

@@ -1717,7 +1717,6 @@ protected function processAdditional()
  $endTokens = [
  T_COLON => true,
  T_COMMA => true,
- T_DOUBLE_ARROW => true,
  T_SEMICOLON => true,
  T_CLOSE_PARENTHESIS => true,
  T_CLOSE_SQUARE_BRACKET => true,

tests/Core/Tokenizer/BackfillFnTokenTest.inc

@@ -51,3 +51,6 @@ fn&($x) => $x;
 $a = [
  'a' => fn() => return 1,
 ];
+
+/* testYield */
+$a = fn($x) => yield 'k' => $x;

tests/Core/Tokenizer/BackfillFnTokenTest.php

@@ -403,6 +403,34 @@ public function testArrayValue()
  }//end testArrayValue()
 
 
+ /**
+ * Test arrow functions that use the yield keyword.
+ *
+ * @covers PHP_CodeSniffer\Tokenizers\PHP::processAdditional
+ *
+ * @return void
+ */
+ public function testYield()
+ {
+ $tokens = self::$phpcsFile->getTokens();
+
+ $token = $this->getTargetToken('/* testYield */', T_FN);
+ $this->backfillHelper($token);
+
+ $this->assertSame($tokens[$token]['scope_opener'], ($token + 5), 'Scope opener is not the arrow token');
+ $this->assertSame($tokens[$token]['scope_closer'], ($token + 14), 'Scope closer is not the semicolon token');
+
+ $opener = $tokens[$token]['scope_opener'];
+ $this->assertSame($tokens[$opener]['scope_opener'], ($token + 5), 'Opener scope opener is not the arrow token');
+ $this->assertSame($tokens[$opener]['scope_closer'], ($token + 14), 'Opener scope closer is not the semicolon token');
+
+ $closer = $tokens[$token]['scope_opener'];
+ $this->assertSame($tokens[$closer]['scope_opener'], ($token + 5), 'Closer scope opener is not the arrow token');
+ $this->assertSame($tokens[$closer]['scope_closer'], ($token + 14), 'Closer scope closer is not the semicolon token');
+
+ }//end testYield()
+
+
  /**
  * Test that anonymous class tokens without parenthesis do not get assigned a parenthesis owner.
  *