Author: Richard Plangger <planri...@gmail.com>
Branch:
Changeset: r82376:eff2a2c4481f
Date: 2016-02-21 22:50 +0100
http://bitbucket.org/pypy/pypy/changeset/eff2a2c4481f/
Log: (mjacob, plan_rich) universal newlines enforced in the tokenizer.
the compile builtin must convert crlf and cr to line feeds. py3.3
import mechanism relies on that
diff --git a/pypy/interpreter/pyparser/pytokenizer.py
b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -91,6 +91,7 @@
strstart = (0, 0, "")
for line in lines:
lnum = lnum + 1
+ line = universal_newline(line)
pos, max = 0, len(line)
if contstr:
@@ -259,3 +260,15 @@
token_list.append((tokens.ENDMARKER, '', lnum, pos, line))
return token_list
+
+def universal_newline(line):
+ if len(line) >= 2:
+ c0 = line[-2]
+ c1 = line[-1]
+ if c0 == '\r' and c1 == '\n':
+ return line[:-2] + '\n'
+ if len(line) >= 1:
+ c = line[-1]
+ if c == '\r':
+ return line[:-1] + '\n'
+ return line
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py
b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -158,3 +158,10 @@
def test_print_function(self):
self.parse("from __future__ import print_function\nx = print\n")
+
+ def test_universal_newlines(self):
+ fmt = 'stuff = """hello%sworld"""'
+ expected_tree = self.parse(fmt % '\n')
+ for linefeed in ["\r\n","\r"]:
+ tree = self.parse(fmt % linefeed)
+ assert expected_tree == tree
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
