Skip to content

Commit 31e3ba0

Browse files
jrfnljrfnl
committed
PHP 8.0: "undo" namespaced names as single token
As per the proposal in 3041. This effectively "undoes" the new PHP 8.0 tokenization of identifier names for PHPCS 3.x. Includes extensive unit tests to ensure the correct re-tokenization as well as that the rest of the tokenization is not adversely affected by this change. Includes preventing `function ...` within a group use statement from breaking the retokenization. Includes fixing the nullable tokenization when combined with any of the new PHP 8 identifier name tokens.
1 parent d33a6a9 commit 31e3ba0

File tree

5 files changed

+1563
-9
lines changed

5 files changed

+1563
-9
lines changed

‎package.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
109109
<file baseinstalldir="" name="StableCommentWhitespaceTest.php" role="test" />
110110
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.inc" role="test" />
111111
<file baseinstalldir="" name="StableCommentWhitespaceWinTest.php" role="test" />
112+
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.inc" role="test" />
113+
<file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.php" role="test" />
112114
</dir>
113115
<file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
114116
<file baseinstalldir="" name="AllTests.php" role="test" />
@@ -1979,6 +1981,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
19791981
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
19801982
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
19811983
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
1984+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
1985+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
19821986
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
19831987
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
19841988
</filelist>
@@ -2038,6 +2042,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20382042
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
20392043
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
20402044
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
2045+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
2046+
<install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
20412047
<install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
20422048
<install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
20432049
<ignore name="bin/phpcs.bat" />

‎src/Tokenizers/PHP.php‎

Lines changed: 100 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -812,6 +812,81 @@ protected function tokenize($string)
812812
continue;
813813
}//end if
814814

815+
/*
816+
As of PHP 8.0 fully qualified, partially qualified and namespace relative
817+
identifier names are tokenized differently.
818+
This "undoes" the new tokenization so the tokenization will be the same in
819+
in PHP 5, 7 and 8.
820+
*/
821+
822+
if (PHP_VERSION_ID >= 80000
823+
&& $tokenIsArray === true
824+
&& ($token[0] === T_NAME_QUALIFIED
825+
|| $token[0] === T_NAME_FULLY_QUALIFIED
826+
|| $token[0] === T_NAME_RELATIVE)
827+
) {
828+
$name = $token[1];
829+
830+
if ($token[0] === T_NAME_FULLY_QUALIFIED) {
831+
$newToken = [];
832+
$newToken['code'] = T_NS_SEPARATOR;
833+
$newToken['type'] = 'T_NS_SEPARATOR';
834+
$newToken['content'] = '\\';
835+
$finalTokens[$newStackPtr] = $newToken;
836+
++$newStackPtr;
837+
838+
$name = ltrim($name, '\\');
839+
}
840+
841+
if ($token[0] === T_NAME_RELATIVE) {
842+
$newToken = [];
843+
$newToken['code'] = T_NAMESPACE;
844+
$newToken['type'] = 'T_NAMESPACE';
845+
$newToken['content'] = substr($name, 0, 9);
846+
$finalTokens[$newStackPtr] = $newToken;
847+
++$newStackPtr;
848+
849+
$newToken = [];
850+
$newToken['code'] = T_NS_SEPARATOR;
851+
$newToken['type'] = 'T_NS_SEPARATOR';
852+
$newToken['content'] = '\\';
853+
$finalTokens[$newStackPtr] = $newToken;
854+
++$newStackPtr;
855+
856+
$name = substr($name, 10);
857+
}
858+
859+
$parts = explode('\\', $name);
860+
$partCount = count($parts);
861+
$lastPart = ($partCount - 1);
862+
863+
foreach ($parts as $i => $part) {
864+
$newToken = [];
865+
$newToken['code'] = T_STRING;
866+
$newToken['type'] = 'T_STRING';
867+
$newToken['content'] = $part;
868+
$finalTokens[$newStackPtr] = $newToken;
869+
++$newStackPtr;
870+
871+
if ($i !== $lastPart) {
872+
$newToken = [];
873+
$newToken['code'] = T_NS_SEPARATOR;
874+
$newToken['type'] = 'T_NS_SEPARATOR';
875+
$newToken['content'] = '\\';
876+
$finalTokens[$newStackPtr] = $newToken;
877+
++$newStackPtr;
878+
}
879+
}
880+
881+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
882+
$type = Util\Tokens::tokenName($token[0]);
883+
$content = Util\Common::prepareForOutput($token[1]);
884+
echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL;
885+
}
886+
887+
continue;
888+
}//end if
889+
815890
/*
816891
Before PHP 7.0, the "yield from" was tokenized as
817892
T_YIELD, T_WHITESPACE and T_STRING. So look for
@@ -1105,7 +1180,7 @@ protected function tokenize($string)
11051180
* Check if the next non-empty token is one of the tokens which can be used
11061181
* in type declarations. If not, it's definitely a ternary.
11071182
* At this point, the only token types which need to be taken into consideration
1108-
* as potential type declarations are T_STRING, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
1183+
* as potential type declarations are identifier names, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
11091184
*/
11101185

11111186
$lastRelevantNonEmpty = null;
@@ -1122,6 +1197,9 @@ protected function tokenize($string)
11221197
}
11231198

11241199
if ($tokenType === T_STRING
1200+
|| $tokenType === T_NAME_FULLY_QUALIFIED
1201+
|| $tokenType === T_NAME_RELATIVE
1202+
|| $tokenType === T_NAME_QUALIFIED
11251203
|| $tokenType === T_ARRAY
11261204
|| $tokenType === T_NS_SEPARATOR
11271205
) {
@@ -1133,7 +1211,10 @@ protected function tokenize($string)
11331211
&& isset($lastRelevantNonEmpty) === false)
11341212
|| ($lastRelevantNonEmpty === T_ARRAY
11351213
&& $tokenType === '(')
1136-
|| ($lastRelevantNonEmpty === T_STRING
1214+
|| (($lastRelevantNonEmpty === T_STRING
1215+
|| $lastRelevantNonEmpty === T_NAME_FULLY_QUALIFIED
1216+
|| $lastRelevantNonEmpty === T_NAME_RELATIVE
1217+
|| $lastRelevantNonEmpty === T_NAME_QUALIFIED)
11371218
&& ($tokenType === T_DOUBLE_COLON
11381219
|| $tokenType === '('
11391220
|| $tokenType === ':'))
@@ -1278,6 +1359,10 @@ protected function tokenize($string)
12781359
tokenized as T_STRING even if it appears to be a different token,
12791360
such as when writing code like: function default(): foo
12801361
so go forward and change the token type before it is processed.
1362+
1363+
Note: this should not be done for `function Level\Name` within a
1364+
group use statement for the PHP 8 identifier name tokens as it
1365+
would interfere with the re-tokenization of those.
12811366
*/
12821367

12831368
if ($tokenIsArray === true
@@ -1295,7 +1380,10 @@ protected function tokenize($string)
12951380
}
12961381
}
12971382

1298-
if ($x < $numTokens && is_array($tokens[$x]) === true) {
1383+
if ($x < $numTokens
1384+
&& is_array($tokens[$x]) === true
1385+
&& $tokens[$x][0] !== T_NAME_QUALIFIED
1386+
) {
12991387
if (PHP_CODESNIFFER_VERBOSITY > 1) {
13001388
$oldType = Util\Tokens::tokenName($tokens[$x][0]);
13011389
echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL;
@@ -1351,12 +1439,15 @@ function return types. We want to keep the parenthesis map clean,
13511439
&& $tokens[$x] === ':'
13521440
) {
13531441
$allowed = [
1354-
T_STRING => T_STRING,
1355-
T_ARRAY => T_ARRAY,
1356-
T_CALLABLE => T_CALLABLE,
1357-
T_SELF => T_SELF,
1358-
T_PARENT => T_PARENT,
1359-
T_NS_SEPARATOR => T_NS_SEPARATOR,
1442+
T_STRING => T_STRING,
1443+
T_NAME_FULLY_QUALIFIED => T_NAME_FULLY_QUALIFIED,
1444+
T_NAME_RELATIVE => T_NAME_RELATIVE,
1445+
T_NAME_QUALIFIED => T_NAME_QUALIFIED,
1446+
T_ARRAY => T_ARRAY,
1447+
T_CALLABLE => T_CALLABLE,
1448+
T_SELF => T_SELF,
1449+
T_PARENT => T_PARENT,
1450+
T_NS_SEPARATOR => T_NS_SEPARATOR,
13601451
];
13611452

13621453
$allowed += Util\Tokens::$emptyTokens;

‎src/Util/Tokens.php‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,19 @@
124124
define('T_FN', 'PHPCS_T_FN');
125125
}
126126

127+
// Some PHP 8.0 tokens, replicated for lower versions.
128+
if (defined('T_NAME_QUALIFIED') === false) {
129+
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
130+
}
131+
132+
if (defined('T_NAME_FULLY_QUALIFIED') === false) {
133+
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
134+
}
135+
136+
if (defined('T_NAME_RELATIVE') === false) {
137+
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
138+
}
139+
127140
// Tokens used for parsing doc blocks.
128141
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
129142
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

‎tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
<?php
2+
3+
/* testNamespaceDeclaration */
4+
namespace Package;
5+
6+
/* testNamespaceDeclarationWithLevels */
7+
namespace Vendor\SubLevel\Domain;
8+
9+
/* testUseStatement */
10+
use ClassName;
11+
12+
/* testUseStatementWithLevels */
13+
use Vendor\Level\Domain;
14+
15+
/* testFunctionUseStatement */
16+
use function function_name;
17+
18+
/* testFunctionUseStatementWithLevels */
19+
use function Vendor\Level\function_in_ns;
20+
21+
/* testConstantUseStatement */
22+
use const CONSTANT_NAME;
23+
24+
/* testConstantUseStatementWithLevels */
25+
use const Vendor\Level\OTHER_CONSTANT;
26+
27+
/* testMultiUseUnqualified */
28+
use UnqualifiedClassName,
29+
/* testMultiUsePartiallyQualified */
30+
Sublevel\PartiallyClassName;
31+
32+
/* testGroupUseStatement */
33+
use Vendor\Level\{
34+
AnotherDomain,
35+
function function_grouped,
36+
const CONSTANT_GROUPED,
37+
Sub\YetAnotherDomain,
38+
function SubLevelA\function_grouped_too,
39+
const SubLevelB\CONSTANT_GROUPED_TOO,
40+
};
41+
42+
/* testClassName */
43+
class MyClass
44+
/* testExtendedFQN */
45+
extends \Vendor\Level\FQN
46+
/* testImplementsRelative */
47+
implements namespace\Name,
48+
/* testImplementsFQN */
49+
\Fully\Qualified,
50+
/* testImplementsUnqualified */
51+
Unqualified,
52+
/* testImplementsPartiallyQualified */
53+
Sub\Level\Name
54+
{
55+
/* testFunctionName */
56+
public function function_name(
57+
/* testTypeDeclarationRelative */
58+
?namespace\Name|object $paramA,
59+
60+
/* testTypeDeclarationFQN */
61+
\Fully\Qualified\Name $paramB,
62+
63+
/* testTypeDeclarationUnqualified */
64+
Unqualified|false $paramC,
65+
66+
/* testTypeDeclarationPartiallyQualified */
67+
?Sublevel\Name $paramD,
68+
69+
/* testReturnTypeFQN */
70+
) : ?\Name {
71+
72+
try {
73+
/* testFunctionCallRelative */
74+
echo NameSpace\function_name();
75+
76+
/* testFunctionCallFQN */
77+
echo \Vendor\Package\function_name();
78+
79+
/* testFunctionCallUnqualified */
80+
echo function_name();
81+
82+
/* testFunctionPartiallyQualified */
83+
echo Level\function_name();
84+
85+
/* testCatchRelative */
86+
} catch (namespace\SubLevel\Exception $e) {
87+
88+
/* testCatchFQN */
89+
} catch (\Exception $e) {
90+
91+
/* testCatchUnqualified */
92+
} catch (Exception $e) {
93+
94+
/* testCatchPartiallyQualified */
95+
} catch (Level\Exception $e) {
96+
}
97+
98+
/* testNewRelative */
99+
$obj = new namespace\ClassName();
100+
101+
/* testNewFQN */
102+
$obj = new \Vendor\ClassName();
103+
104+
/* testNewUnqualified */
105+
$obj = new ClassName;
106+
107+
/* testNewPartiallyQualified */
108+
$obj = new Level\ClassName;
109+
110+
/* testDoubleColonRelative */
111+
$value = namespace\ClassName::property;
112+
113+
/* testDoubleColonFQN */
114+
$value = \ClassName::static_function();
115+
116+
/* testDoubleColonUnqualified */
117+
$value = ClassName::CONSTANT_NAME;
118+
119+
/* testDoubleColonPartiallyQualified */
120+
$value = Level\ClassName::CONSTANT_NAME['key'];
121+
122+
/* testInstanceOfRelative */
123+
$is = $obj instanceof namespace\ClassName;
124+
125+
/* testInstanceOfFQN */
126+
if ($obj instanceof \Full\ClassName) {}
127+
128+
/* testInstanceOfUnqualified */
129+
if ($a === $b && $obj instanceof ClassName && true) {}
130+
131+
/* testInstanceOfPartiallyQualified */
132+
$is = $obj instanceof Partially\ClassName;
133+
}
134+
}
135+
136+
/* testInvalidInPHP8Whitespace */
137+
namespace \ Sublevel
138+
\ function_name();
139+
140+
/* testInvalidInPHP8Comments */
141+
$value = \Fully
142+
// phpcs:ignore Stnd.Cat.Sniff -- for reasons
143+
\Qualified
144+
/* comment */
145+
\Name
146+
// comment
147+
:: function_name();

0 commit comments

Comments
 (0)