◐ Shell
reader mode source ↗
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
File filter
Conversations
Jump to
Diff view
Apply and reload
Show whitespace
Diff view
Apply and reload
24 changes: 23 additions & 1 deletion Lib/test/test_tokenize.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from test import support
from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
open as tokenize_open, Untokenizer)
from io import BytesIO
from unittest import TestCase, mock
from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
Expand Down Expand Up @@ -1351,6 +1351,28 @@ def assertExactTypeEqual(self, opstr, *optypes):
self.assertEqual(tok_name[tokens[1 + num_optypes].exact_type],
tok_name[token.ENDMARKER])

def test_exact_type(self):
self.assertExactTypeEqual('()', token.LPAR, token.RPAR)
self.assertExactTypeEqual('[]', token.LSQB, token.RSQB)
Expand Down
22 changes: 13 additions & 9 deletions Lib/token.py
Original file line number Diff line number Diff line change
@@ -129,33 +129,37 @@ def _main():
comment = comment_match.group(1)
tokens[prev_val]['comment'] = comment
keys = sorted(tokens.keys())
# load the output skeleton from the target:
try:
fp = open(outFileName)
except OSError as err:
sys.stderr.write("I/O error: %s\n" % str(err))
sys.exit(2)
with fp:
format = fp.read().split("\n")
try:
start = format.index("#--start constants--") + 1
end = format.index("#--end constants--")
except ValueError:
sys.stderr.write("target does not contain format markers")
sys.exit(3)
lines = []
for key in keys:
lines.append("%s = %d" % (tokens[key]["token"], key))
if "comment" in tokens[key]:
lines.append("# %s" % tokens[key]["comment"])
format[start:end] = lines
try:
fp = open(outFileName, 'w')
except OSError as err:
sys.stderr.write("I/O error: %s\n" % str(err))
sys.exit(4)
with fp:
fp.write("\n".join(format))


if __name__ == "__main__":
Expand Down
Toggle all file notes Toggle all file annotations