Irit Katriel <[email protected]> added the comment:
For reference - this script:
---------------------------------------------
import inspect
import tokenize
from pprint import pprint as pp
src=[
'def f():\n',
' return 1\n',
' #that was fun',
'\n',
'#Now comes g\n',
'def g():\n',
' return 2\n']
pp(list(tokenize.generate_tokens(iter(src).__next__)))
---------------------------------------------
Outputs:
---------------------------------------------
[TokenInfo(type=1 (NAME), string='def', start=(1, 0), end=(1, 3), line='def
f():\n'),
TokenInfo(type=1 (NAME), string='f', start=(1, 4), end=(1, 5), line='def
f():\n'),
TokenInfo(type=54 (OP), string='(', start=(1, 5), end=(1, 6), line='def
f():\n'),
TokenInfo(type=54 (OP), string=')', start=(1, 6), end=(1, 7), line='def
f():\n'),
TokenInfo(type=54 (OP), string=':', start=(1, 7), end=(1, 8), line='def
f():\n'),
TokenInfo(type=4 (NEWLINE), string='\n', start=(1, 8), end=(1, 9), line='def
f():\n'),
TokenInfo(type=5 (INDENT), string=' ', start=(2, 0), end=(2, 4), line='
return 1\n'),
TokenInfo(type=1 (NAME), string='return', start=(2, 4), end=(2, 10), line='
return 1\n'),
TokenInfo(type=2 (NUMBER), string='1', start=(2, 11), end=(2, 12), line='
return 1\n'),
TokenInfo(type=4 (NEWLINE), string='\n', start=(2, 12), end=(2, 13), line='
return 1\n'),
TokenInfo(type=60 (COMMENT), string='#that was fun', start=(3, 4), end=(3,
17), line=' #that was fun'),
TokenInfo(type=61 (NL), string='', start=(3, 17), end=(3, 17), line=' #that
was fun'),
TokenInfo(type=61 (NL), string='\n', start=(4, 0), end=(4, 1), line='\n'),
TokenInfo(type=60 (COMMENT), string='#Now comes g', start=(5, 0), end=(5, 12),
line='#Now comes g\n'),
TokenInfo(type=61 (NL), string='\n', start=(5, 12), end=(5, 13), line='#Now
comes g\n'),
TokenInfo(type=6 (DEDENT), string='', start=(6, 0), end=(6, 0), line='def
g():\n'),
TokenInfo(type=1 (NAME), string='def', start=(6, 0), end=(6, 3), line='def
g():\n'),
TokenInfo(type=1 (NAME), string='g', start=(6, 4), end=(6, 5), line='def
g():\n'),
TokenInfo(type=54 (OP), string='(', start=(6, 5), end=(6, 6), line='def
g():\n'),
TokenInfo(type=54 (OP), string=')', start=(6, 6), end=(6, 7), line='def
g():\n'),
TokenInfo(type=54 (OP), string=':', start=(6, 7), end=(6, 8), line='def
g():\n'),
TokenInfo(type=4 (NEWLINE), string='\n', start=(6, 8), end=(6, 9), line='def
g():\n'),
TokenInfo(type=5 (INDENT), string=' ', start=(7, 0), end=(7, 4), line='
return 2\n'),
TokenInfo(type=1 (NAME), string='return', start=(7, 4), end=(7, 10), line='
return 2\n'),
TokenInfo(type=2 (NUMBER), string='2', start=(7, 11), end=(7, 12), line='
return 2\n'),
TokenInfo(type=4 (NEWLINE), string='\n', start=(7, 12), end=(7, 13), line='
return 2\n'),
TokenInfo(type=6 (DEDENT), string='', start=(8, 0), end=(8, 0), line=''),
TokenInfo(type=0 (ENDMARKER), string='', start=(8, 0), end=(8, 0), line='')]
----------
_______________________________________
Python tracker <[email protected]>
<https://bugs.python.org/issue42116>
_______________________________________
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
https://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
