[3.7] Revert "bpo-30406: Make async and await proper keywords" (GH-6143) by miss-islington · Pull Request #6145 · python/cpython

Expand Up @@ -234,7 +234,7 @@ def compat(self, token, iterable): for tok in iterable: toknum, tokval = tok[:2]
if toknum in (NAME, NUMBER): if toknum in (NAME, NUMBER, ASYNC, AWAIT): tokval += ' '
if toknum == INDENT: Expand Down Expand Up @@ -380,6 +380,12 @@ def generate_tokens(readline): contline = None indents = [0]
# 'stashed' and 'async_*' are used for async/await parsing stashed = None async_def = False async_def_indent = 0 async_def_nl = False
while 1: # loop over lines in stream try: line = readline() Expand Down Expand Up @@ -420,6 +426,10 @@ def generate_tokens(readline): pos = pos + 1 if pos == max: break
if stashed: yield stashed stashed = None
if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') Expand All @@ -443,8 +453,18 @@ def generate_tokens(readline): ("<tokenize>", lnum, pos, line)) indents = indents[:-1]
if async_def and async_def_indent >= indents[-1]: async_def = False async_def_nl = False async_def_indent = 0
yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
if async_def and async_def_nl and async_def_indent >= indents[-1]: async_def = False async_def_nl = False async_def_indent = 0
else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) Expand All @@ -464,17 +484,28 @@ def generate_tokens(readline): newline = NEWLINE if parenlev > 0: newline = NL elif async_def: async_def_nl = True if stashed: yield stashed stashed = None yield (newline, token, spos, epos, line)
elif initial == '#': assert not token.endswith("\n") if stashed: yield stashed stashed = None yield (COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] if stashed: yield stashed stashed = None yield (STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines Expand All @@ -492,22 +523,63 @@ def generate_tokens(readline): contline = line break else: # ordinary string if stashed: yield stashed stashed = None yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name yield (NAME, token, spos, epos, line) if token in ('async', 'await'): if async_def: yield (ASYNC if token == 'async' else AWAIT, token, spos, epos, line) continue
tok = (NAME, token, spos, epos, line) if token == 'async' and not stashed: stashed = tok continue
if token == 'def': if (stashed and stashed[0] == NAME and stashed[1] == 'async'):
async_def = True async_def_indent = indents[-1]
yield (ASYNC, stashed[1], stashed[2], stashed[3], stashed[4]) stashed = None
if stashed: yield stashed stashed = None
yield tok elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: if stashed: yield stashed stashed = None yield (NL, token, spos, (lnum, pos), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 if stashed: yield stashed stashed = None yield (OP, token, spos, epos, line) else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos = pos + 1
if stashed: yield stashed stashed = None
for indent in indents[1:]: # pop remaining indent levels yield (DEDENT, '', (lnum, 0), (lnum, 0), '') yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') Expand Down