[pypy-commit] pypy py3.5: Use utf-8 for readline history file.

[exarkun]

Author: Jean-Paul Calderone <exar...@twistedmatrix.com>
Branch: py3.5
Changeset: r92145:759e514cd5ab
Date: 2017-08-14 11:23 -0400
http://bitbucket.org/pypy/pypy/changeset/759e514cd5ab/

Log:    Use utf-8 for readline history file.

diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -297,10 +297,7 @@
         line = line.rstrip('\n')
         if isinstance(line, unicode):
             return line # on py3k
-        try:
-            return unicode(line, ENCODING)
-        except UnicodeDecodeError:   # bah, silently fall back...
-            return unicode(line, 'utf-8', 'replace')
+        return unicode(line, 'utf-8', 'replace')
 
     def get_history_length(self):
         return self.saved_history_length
@@ -317,7 +314,7 @@
         # history item: we use \r\n instead of just \n.  If the history
         # file is passed to GNU readline, the extra \r are just ignored.
         history = self.get_reader().history
-        f = open(os.path.expanduser(filename), 'r')
+        f = open(os.path.expanduser(filename), 'r', encoding='utf-8')
         buffer = []
         for line in f:
             if line.endswith('\r\n'):
@@ -334,15 +331,12 @@
     def write_history_file(self, filename='~/.history'):
         maxlength = self.saved_history_length
         history = self.get_reader().get_trimmed_history(maxlength)
-        f = open(os.path.expanduser(filename), 'w')
+        f = open(os.path.expanduser(filename), 'w', encoding='utf-8')
         for entry in history:
             # if we are on py3k, we don't need to encode strings before
             # writing it to a file
             if isinstance(entry, unicode) and sys.version_info < (3,):
-                try:
-                    entry = entry.encode(ENCODING)
-                except UnicodeEncodeError:   # bah, silently fall back...
-                    entry = entry.encode('utf-8')
+                entry = entry.encode('utf-8')
             entry = entry.replace('\n', '\r\n')   # multiline history support
             f.write(entry + '\n')
         f.close()
diff --git a/pypy/module/readline/test/test_readline.py 
b/pypy/module/readline/test/test_readline.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/readline/test/test_readline.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+class AppTestReadline:
+    spaceconfig = dict(usemodules={
+        'unicodedata', 'termios', 'select', 'signal', 'fcntl',
+        '_minimal_curses', 'faulthandler', '_socket', 'binascii',
+        '_posixsubprocess',
+    })
+
+    def test_nonascii_history(self):
+        import os, readline
+        TESTFN = "{}_{}_tmp".format("@test", os.getpid())
+
+        is_editline = readline.__doc__ and "libedit" in readline.__doc__
+
+        readline.clear_history()
+        try:
+            readline.add_history("entr&#233;e 1")
+        except UnicodeEncodeError as err:
+            skip("Locale cannot encode test data: " + format(err))
+        readline.add_history("entr&#233;e 2")
+        readline.replace_history_item(1, "entr&#233;e 22")
+        readline.write_history_file(TESTFN)
+        readline.clear_history()
+        readline.read_history_file(TESTFN)
+        if is_editline:
+            # An add_history() call seems to be required for get_history_
+            # item() to register items from the file
+            readline.add_history("dummy")
+        assert readline.get_history_item(1) ==  "entr&#233;e 1"
+        assert readline.get_history_item(2) == "entr&#233;e 22"
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit