Skip to content

Commit a90df50

Browse files
miss-islingtonJelleZijlstra
miss-islington
and
JelleZijlstra
authored
Revert "bpo-30406: Make async and await proper keywords (GH-1669)" (GH-6143)
This reverts commit ac31770. (Reverts only the lib2to3 part.) (cherry picked from commit f64aae4) Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
1 parent 773573e commit a90df50

File tree

4 files changed

+94
-20
lines changed

4 files changed

+94
-20
lines changed

‎Lib/lib2to3/Grammar.txt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ eval_input: testlist NEWLINE* ENDMARKER
1515
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
1616
decorators: decorator+
1717
decorated: decorators (classdef | funcdef | async_funcdef)
18-
async_funcdef: 'async' funcdef
18+
async_funcdef: ASYNC funcdef
1919
funcdef: 'def' NAME parameters ['->' test] ':' suite
2020
parameters: '(' [typedargslist] ')'
2121
typedargslist: ((tfpdef ['=' test] ',')*
@@ -66,7 +66,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]]
6666
assert_stmt: 'assert' test [',' test]
6767

6868
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
69-
async_stmt: 'async' (funcdef | with_stmt | for_stmt)
69+
async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
7070
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
7171
while_stmt: 'while' test ':' suite ['else' ':' suite]
7272
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -105,7 +105,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
105105
arith_expr: term (('+'|'-') term)*
106106
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
107107
factor: ('+'|'-'|'~') factor | power
108-
power: ['await'] atom trailer* ['**' factor]
108+
power: [AWAIT] atom trailer* ['**' factor]
109109
atom: ('(' [yield_expr|testlist_gexp] ')' |
110110
'[' [listmaker] ']' |
111111
'{' [dictsetmaker] '}' |
@@ -142,7 +142,7 @@ argument: ( test [comp_for] |
142142
star_expr )
143143

144144
comp_iter: comp_for | comp_if
145-
comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
145+
comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
146146
comp_if: 'if' old_test [comp_iter]
147147

148148
# As noted above, testlist_safe extends the syntax allowed in list
@@ -161,7 +161,7 @@ comp_if: 'if' old_test [comp_iter]
161161
#
162162
# See https://bugs.python.org/issue27494
163163
old_comp_iter: old_comp_for | old_comp_if
164-
old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter]
164+
old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter]
165165
old_comp_if: 'if' old_test [old_comp_iter]
166166

167167
testlist1: test (',' test)*

‎Lib/lib2to3/pgen2/token.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,10 @@
6262
COMMENT = 53
6363
NL = 54
6464
RARROW = 55
65-
ERRORTOKEN = 56
66-
N_TOKENS = 57
65+
AWAIT = 56
66+
ASYNC = 57
67+
ERRORTOKEN = 58
68+
N_TOKENS = 59
6769
NT_OFFSET = 256
6870
#--end constants--
6971

‎Lib/lib2to3/pgen2/tokenize.py‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ def compat(self, token, iterable):
234234
for tok in iterable:
235235
toknum, tokval = tok[:2]
236236

237-
if toknum in (NAME, NUMBER):
237+
if toknum in (NAME, NUMBER, ASYNC, AWAIT):
238238
tokval += ' '
239239

240240
if toknum == INDENT:
@@ -380,6 +380,12 @@ def generate_tokens(readline):
380380
contline = None
381381
indents = [0]
382382

383+
# 'stashed' and 'async_*' are used for async/await parsing
384+
stashed = None
385+
async_def = False
386+
async_def_indent = 0
387+
async_def_nl = False
388+
383389
while 1: # loop over lines in stream
384390
try:
385391
line = readline()
@@ -420,6 +426,10 @@ def generate_tokens(readline):
420426
pos = pos + 1
421427
if pos == max: break
422428

429+
if stashed:
430+
yield stashed
431+
stashed = None
432+
423433
if line[pos] in '#\r\n': # skip comments or blank lines
424434
if line[pos] == '#':
425435
comment_token = line[pos:].rstrip('\r\n')
@@ -443,8 +453,18 @@ def generate_tokens(readline):
443453
("<tokenize>", lnum, pos, line))
444454
indents = indents[:-1]
445455

456+
if async_def and async_def_indent >= indents[-1]:
457+
async_def = False
458+
async_def_nl = False
459+
async_def_indent = 0
460+
446461
yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
447462

463+
if async_def and async_def_nl and async_def_indent >= indents[-1]:
464+
async_def = False
465+
async_def_nl = False
466+
async_def_indent = 0
467+
448468
else: # continued statement
449469
if not line:
450470
raise TokenError("EOF in multi-line statement", (lnum, 0))
@@ -464,17 +484,28 @@ def generate_tokens(readline):
464484
newline = NEWLINE
465485
if parenlev > 0:
466486
newline = NL
487+
elif async_def:
488+
async_def_nl = True
489+
if stashed:
490+
yield stashed
491+
stashed = None
467492
yield (newline, token, spos, epos, line)
468493

469494
elif initial == '#':
470495
assert not token.endswith("\n")
496+
if stashed:
497+
yield stashed
498+
stashed = None
471499
yield (COMMENT, token, spos, epos, line)
472500
elif token in triple_quoted:
473501
endprog = endprogs[token]
474502
endmatch = endprog.match(line, pos)
475503
if endmatch: # all on one line
476504
pos = endmatch.end(0)
477505
token = line[start:pos]
506+
if stashed:
507+
yield stashed
508+
stashed = None
478509
yield (STRING, token, spos, (lnum, pos), line)
479510
else:
480511
strstart = (lnum, start) # multiple lines
@@ -492,22 +523,63 @@ def generate_tokens(readline):
492523
contline = line
493524
break
494525
else: # ordinary string
526+
if stashed:
527+
yield stashed
528+
stashed = None
495529
yield (STRING, token, spos, epos, line)
496530
elif initial in namechars: # ordinary name
497-
yield (NAME, token, spos, epos, line)
531+
if token in ('async', 'await'):
532+
if async_def:
533+
yield (ASYNC if token == 'async' else AWAIT,
534+
token, spos, epos, line)
535+
continue
536+
537+
tok = (NAME, token, spos, epos, line)
538+
if token == 'async' and not stashed:
539+
stashed = tok
540+
continue
541+
542+
if token == 'def':
543+
if (stashed
544+
and stashed[0] == NAME
545+
and stashed[1] == 'async'):
546+
547+
async_def = True
548+
async_def_indent = indents[-1]
549+
550+
yield (ASYNC, stashed[1],
551+
stashed[2], stashed[3],
552+
stashed[4])
553+
stashed = None
554+
555+
if stashed:
556+
yield stashed
557+
stashed = None
558+
559+
yield tok
498560
elif initial == '\\': # continued stmt
499561
# This yield is new; needed for better idempotency:
562+
if stashed:
563+
yield stashed
564+
stashed = None
500565
yield (NL, token, spos, (lnum, pos), line)
501566
continued = 1
502567
else:
503568
if initial in '([{': parenlev = parenlev + 1
504569
elif initial in ')]}': parenlev = parenlev - 1
570+
if stashed:
571+
yield stashed
572+
stashed = None
505573
yield (OP, token, spos, epos, line)
506574
else:
507575
yield (ERRORTOKEN, line[pos],
508576
(lnum, pos), (lnum, pos+1), line)
509577
pos = pos + 1
510578

579+
if stashed:
580+
yield stashed
581+
stashed = None
582+
511583
for indent in indents[1:]: # pop remaining indent levels
512584
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
513585
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')

‎Lib/lib2to3/tests/test_parser.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -181,34 +181,34 @@ def foo(): pass
181181
async def foo(): await x
182182
""")
183183

184-
self.validate("await x")
185-
self.validate("""def foo():
186-
await x""")
184+
self.invalid_syntax("await x")
185+
self.invalid_syntax("""def foo():
186+
await x""")
187187

188-
self.validate("""def foo():
188+
self.invalid_syntax("""def foo():
189189
def foo(): pass
190190
async def foo(): pass
191191
await x
192192
""")
193193

194194
def test_async_var(self):
195-
self.invalid_syntax("""async = 1""")
196-
self.invalid_syntax("""await = 1""")
197-
self.invalid_syntax("""def async(): pass""")
195+
self.validate("""async = 1""")
196+
self.validate("""await = 1""")
197+
self.validate("""def async(): pass""")
198198

199199
def test_async_with(self):
200200
self.validate("""async def foo():
201201
async for a in b: pass""")
202202

203-
self.validate("""def foo():
204-
async for a in b: pass""")
203+
self.invalid_syntax("""def foo():
204+
async for a in b: pass""")
205205

206206
def test_async_for(self):
207207
self.validate("""async def foo():
208208
async with a: pass""")
209209

210-
self.validate("""def foo():
211-
async with a: pass""")
210+
self.invalid_syntax("""def foo():
211+
async with a: pass""")
212212

213213

214214
class TestRaiseChanges(GrammarTest):

0 commit comments

Comments
 (0)