Author: Matti Picus <[email protected]>
Branch: py3.5-mac-embedding
Changeset: r92830:7693d7783f04
Date: 2017-10-22 18:56 +0300
http://bitbucket.org/pypy/pypy/changeset/7693d7783f04/
Log: merge py3.5 into branch
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -76,6 +76,8 @@
^lib_pypy/.+.c$
^lib_pypy/.+.o$
^lib_pypy/.+.so$
+^lib_pypy/.+.pyd$
+^lib_pypy/Release/
^pypy/doc/discussion/.+\.html$
^include/.+\.h$
^include/.+\.inl$
diff --git a/lib-python/3/ctypes/test/test_bitfields.py
b/lib-python/3/ctypes/test/test_bitfields.py
--- a/lib-python/3/ctypes/test/test_bitfields.py
+++ b/lib-python/3/ctypes/test/test_bitfields.py
@@ -1,5 +1,5 @@
from ctypes import *
-from ctypes.test import need_symbol
+from ctypes.test import need_symbol, xfail
import unittest
import os
@@ -279,6 +279,7 @@
self.assertEqual(b, b'\xef\xcd\xab\x21')
@need_symbol('c_uint32')
+ @xfail
def test_uint32_swap_big_endian(self):
# Issue #23319
class Big(BigEndianStructure):
diff --git a/lib-python/3/ctypes/test/test_byteswap.py
b/lib-python/3/ctypes/test/test_byteswap.py
--- a/lib-python/3/ctypes/test/test_byteswap.py
+++ b/lib-python/3/ctypes/test/test_byteswap.py
@@ -2,6 +2,7 @@
from binascii import hexlify
from ctypes import *
+from test.support import impl_detail
def bin(s):
return hexlify(memoryview(s)).decode().upper()
@@ -22,6 +23,7 @@
setattr(bits, "i%s" % i, 1)
dump(bits)
+ @impl_detail("slots are irrelevant on PyPy", pypy=False)
def test_slots(self):
class BigPoint(BigEndianStructure):
__slots__ = ()
diff --git a/lib-python/3/ctypes/test/test_frombuffer.py
b/lib-python/3/ctypes/test/test_frombuffer.py
--- a/lib-python/3/ctypes/test/test_frombuffer.py
+++ b/lib-python/3/ctypes/test/test_frombuffer.py
@@ -85,7 +85,6 @@
del a
gc.collect() # Should not crash
- @xfail
def test_from_buffer_copy(self):
a = array.array("i", range(16))
x = (c_int * 16).from_buffer_copy(a)
diff --git a/lib-python/3/test/test_bytes.py b/lib-python/3/test/test_bytes.py
--- a/lib-python/3/test/test_bytes.py
+++ b/lib-python/3/test/test_bytes.py
@@ -721,9 +721,12 @@
self.assertIs(type(BytesSubclass(A())), BytesSubclass)
# Test PyBytes_FromFormat()
- @test.support.impl_detail("don't test cpyext here")
def test_from_format(self):
test.support.import_module('ctypes')
+ try:
+ from ctypes import pythonapi
+ except ImportError:
+ self.skipTest( "no pythonapi in ctypes")
from ctypes import pythonapi, py_object, c_int, c_char_p
PyBytes_FromFormat = pythonapi.PyBytes_FromFormat
PyBytes_FromFormat.restype = py_object
diff --git a/lib-python/3/test/test_unicode.py
b/lib-python/3/test/test_unicode.py
--- a/lib-python/3/test/test_unicode.py
+++ b/lib-python/3/test/test_unicode.py
@@ -2396,6 +2396,10 @@
# Test PyUnicode_FromFormat()
def test_from_format(self):
support.import_module('ctypes')
+ try:
+ from ctypes import pythonapi
+ except ImportError:
+ self.skipTest( "no pythonapi in ctypes")
from ctypes import (
pythonapi, py_object, sizeof,
c_int, c_long, c_longlong, c_ssize_t,
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -8,9 +8,14 @@
class ArrayMeta(_CDataMeta):
def __new__(self, name, cls, typedict):
res = type.__new__(self, name, cls, typedict)
+
if cls == (_CData,): # this is the Array class defined below
+ res._ffiarray = None
return res
-
+ if not hasattr(res, '_length_') or not isinstance(res._length_, int):
+ raise AttributeError(
+ "class must define a '_length_' attribute, "
+ "which must be a positive integer")
ffiarray = res._ffiarray = _rawffi.Array(res._type_._ffishape_)
subletter = getattr(res._type_, '_type_', None)
if subletter == 'c':
@@ -55,7 +60,7 @@
for i in range(len(val)):
target[i] = val[i]
if len(val) < self._length_:
- target[len(val)] = '\x00'
+ target[len(val)] = u'\x00'
res.value = property(getvalue, setvalue)
res._ffishape_ = (ffiarray, res._length_)
@@ -164,7 +169,7 @@
if letter == 'c':
return b"".join(l)
if letter == 'u':
- return "".join(l)
+ return u"".join(l)
return l
class Array(_CData, metaclass=ArrayMeta):
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -232,9 +232,6 @@
elif tp == 'u':
def _setvalue(self, val):
- if isinstance(val, bytes):
- val = val.decode(ConvMode.encoding, ConvMode.errors)
- # possible if we use 'ignore'
if val:
self._buffer[0] = val
def _getvalue(self):
@@ -243,8 +240,6 @@
elif tp == 'c':
def _setvalue(self, val):
- if isinstance(val, str):
- val = val.encode(ConvMode.encoding, ConvMode.errors)
if val:
self._buffer[0] = val
def _getvalue(self):
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.11.1
+Version: 1.11.2
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI
from .error import CDefError, FFIError, VerificationError, VerificationMissing
-__version__ = "1.11.1"
-__version_info__ = (1, 11, 1)
+__version__ = "1.11.2"
+__version_info__ = (1, 11, 2)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -238,9 +238,9 @@
_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
{
if (sizeof(_cffi_wchar_t) == 2)
- return _cffi_from_c_wchar_t(x);
+ return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
else
- return _cffi_from_c_wchar3216_t(x);
+ return _cffi_from_c_wchar3216_t((int)x);
}
_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
@@ -254,7 +254,7 @@
_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
{
if (sizeof(_cffi_wchar_t) == 4)
- return _cffi_from_c_wchar_t(x);
+ return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
else
return _cffi_from_c_wchar3216_t(x);
}
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -247,7 +247,7 @@
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.11.1"
+ "\ncompiled with cffi version: 1.11.2"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/TODO b/pypy/TODO
--- a/pypy/TODO
+++ b/pypy/TODO
@@ -1,18 +1,4 @@
-TODO for the python3 test suite:
-
-* test_memoryview
- Needs bytes/str changes. Probably easy. Work for this has begun on
- py3k-memoryview (by mjacob) https://bugs.pypy.org/issue1542
-
-own-tests:
-
-* module/test_lib_pypy
- These crash the buildbots (via SyntaxErrors): others were really
- made to run under Python 2.x and so simply fail
-
-* module.cpyext.test.test_structseq test_StructSeq
- structseq now subclasses tuple on py3, which breaks how
- BaseCpyTypeDescr.realize allocates it
+...
antocuni's older TODO:
@@ -20,14 +6,6 @@
* run coverage against the parser/astbuilder/astcompiler: it's probably full of
dead code because the grammar changed
-* re-enable strategies https://bugs.pypy.org/issue1540 :
- - re-enable IntDictStrategy
- - re-enable StdObjSpace.listview_str
- - re-enable the kwargs dict strategy in dictmultiobject.py
- - re-enable view_as_kwargs
-
-* unskip numpypy tests in module/test_lib_pypy/numpypy/
-
* optimize W_UnicodeObject, right now it stores both an unicode and an utf8
version of the same string
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -119,7 +119,7 @@
To run untranslated tests, you need the Boehm garbage collector libgc.
-On recent Debian and Ubuntu (like 17.04), this is the command to install
+On recent Debian and Ubuntu (16.04 onwards), this is the command to install
all build-time dependencies::
apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \
@@ -127,7 +127,7 @@
tk-dev libgc-dev python-cffi \
liblzma-dev libncursesw5-dev # these two only needed on PyPy3
-On older Debian and Ubuntu (12.04 to 16.04)::
+On older Debian and Ubuntu (12.04-14.04)::
apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
@@ -149,12 +149,23 @@
xz-devel # For lzma on PyPy3.
(XXX plus the SLES11 version of libgdbm-dev and tk-dev)
-On Mac OS X, most of these build-time dependencies are installed alongside
+On Mac OS X::
+
+Most of these build-time dependencies are installed alongside
the Developer Tools. However, note that in order for the installation to
find them you may need to run::
xcode-select --install
+An exception is OpenSSL, which is no longer provided with the operating
+system. It can be obtained via Homebrew (with ``$ brew install openssl``),
+but it will not be available on the system path by default. The easiest
+way to enable it for building pypy is to set an environment variable::
+
+ export PKG_CONFIG_PATH=$(brew --prefix)/opt/openssl/lib/pkgconfig
+
+After setting this, translation (described next) will find the OpenSSL libs
+as expected.
Run the translation
-------------------
@@ -187,18 +198,18 @@
entire pypy interpreter. This step is currently singe threaded, and RAM
hungry. As part of this step, the chain creates a large number of C code
files and a Makefile to compile them in a
- directory controlled by the ``PYPY_USESSION_DIR`` environment variable.
+ directory controlled by the ``PYPY_USESSION_DIR`` environment variable.
2. Create an executable ``pypy-c`` by running the Makefile. This step can
- utilize all possible cores on the machine.
-3. Copy the needed binaries to the current directory.
-4. Generate c-extension modules for any cffi-based stdlib modules.
+ utilize all possible cores on the machine.
+3. Copy the needed binaries to the current directory.
+4. Generate c-extension modules for any cffi-based stdlib modules.
The resulting executable behaves mostly like a normal Python
interpreter (see :doc:`cpython_differences`), and is ready for testing, for
use as a base interpreter for a new virtualenv, or for packaging into a binary
suitable for installation on another machine running the same OS as the build
-machine.
+machine.
Note that step 4 is merely done as a convenience, any of the steps may be rerun
without rerunning the previous steps.
@@ -255,7 +266,7 @@
* PyPy 2.5.1 or earlier: normal users would see permission errors.
Installers need to run ``pypy -c "import gdbm"`` and other similar
- commands at install time; the exact list is in
+ commands at install time; the exact list is in
:source:`pypy/tool/release/package.py <package.py>`. Users
seeing a broken installation of PyPy can fix it after-the-fact if they
have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -240,9 +240,12 @@
**matplotlib** https://github.com/matplotlib/matplotlib
- TODO: the tkagg backend does not work, which makes tests fail on downstream
- projects like Pandas, SciPy. It uses id(obj) as a c-pointer to obj in
- tkagg.py, which requires refactoring
+ Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of
+ matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the
+ tkagg backend can function.
+
+ TODO: the matplotlib branch passes numpy arrays by value (copying all the
+ data), this proof-of-concept needs help to become completely compliant
**wxPython** https://bitbucket.org/amauryfa/wxpython-cffi
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -89,7 +89,7 @@
startrev, documented = parse_doc(last_whatsnew)
merged, branch = get_merged_branches(ROOT, startrev, '')
merged.discard('default')
- merged.discard('py3k')
+ merged.discard('py3.5')
merged.discard('')
not_documented = merged.difference(documented)
not_merged = documented.difference(merged)
@@ -100,7 +100,7 @@
print '\n'.join(not_merged)
print
assert not not_documented
- if branch == 'py3k':
+ if branch == 'py3.5':
assert not not_merged
else:
assert branch in documented, 'Please document this branch before
merging: %s' % branch
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,8 @@
.. this is a revision shortly after release-pypy2.7-v5.9.0
.. startrev:d56dadcef996
+.. branch: cppyy-packaging
+Cleanup and improve cppyy packaging
+.. branch: docs-osx-brew-openssl
+
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -4,8 +4,3 @@
.. this is the revision after release-pypy3.5-5.9
.. startrev: be41e3ac0a29
-
-.. branch: multiphase
-
-Implement PyType_FromSpec (PEP 384) and fix issues with PEP 489 support.
-
diff --git a/pypy/module/_cffi_backend/__init__.py
b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.11.1"
+VERSION = "1.11.2"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.11.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.2", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
diff --git a/pypy/module/_codecs/test/test_codecs.py
b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -762,9 +762,16 @@
assert s == b'\xe9'
def test_lone_surrogates(self):
- for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
- 'utf-32', 'utf-32-le', 'utf-32-be'):
+ encodings = ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
+ 'utf-32', 'utf-32-le', 'utf-32-be')
+ for encoding in encodings:
raises(UnicodeEncodeError, u'\ud800'.encode, encoding)
+ assert (u'[\udc80]'.encode(encoding, "backslashreplace") ==
+ '[\\udc80]'.encode(encoding))
+ assert (u'[\udc80]'.encode(encoding, "ignore") ==
+ '[]'.encode(encoding))
+ assert (u'[\udc80]'.encode(encoding, "replace") ==
+ '[?]'.encode(encoding))
def test_charmap_encode(self):
assert 'xxx'.encode('charmap') == b'xxx'
diff --git a/pypy/module/_socket/test/test_sock_app.py
b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -574,6 +574,22 @@
import _socket
raises(_socket.error, _socket.dup, 123456)
+ def test_recvmsg_issue2649(self):
+ import _socket as socket
+ listener = socket.socket(family=socket.AF_INET6,
type=socket.SOCK_DGRAM)
+ listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ listener.bind(('::1', 1234))
+
+ s = socket.socket(family=socket.AF_INET6, type=socket.SOCK_DGRAM)
+ IPV6_RECVERR = 25
+ s.setsockopt(socket.IPPROTO_IPV6, IPV6_RECVERR, 1)
+
+ s.sendto(b'x', ('::1', 1234))
+ try:
+ queue = s.recvmsg(1024, 1024, socket.MSG_ERRQUEUE)
+ except BlockingIOError as e:
+ assert True
+
def test_buffer(self):
# Test that send/sendall/sendto accept a buffer as arg
import _socket, os
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -274,7 +274,10 @@
if pos == 0:
# Store the current keys in the PyDictObject.
decref(space, py_dict.c__tmpkeys)
- w_keys = space.call_method(space.w_dict, "keys", w_dict)
+ w_keyview = space.call_method(space.w_dict, "keys", w_dict)
+ # w_keys must use the object strategy in order to keep the keys alive
+ w_keys = space.newlist(space.listview(w_keyview))
+ w_keys.switch_to_object_strategy()
py_dict.c__tmpkeys = create_ref(space, w_keys)
Py_IncRef(space, py_dict.c__tmpkeys)
else:
@@ -287,10 +290,10 @@
decref(space, py_dict.c__tmpkeys)
py_dict.c__tmpkeys = lltype.nullptr(PyObject.TO)
return 0
- w_key = space.listview(w_keys)[pos]
+ w_key = space.listview(w_keys)[pos] # fast iff w_keys uses object strat
w_value = space.getitem(w_dict, w_key)
if pkey:
- pkey[0] = as_pyobj(space, w_key)
+ pkey[0] = as_pyobj(space, w_key)
if pvalue:
pvalue[0] = as_pyobj(space, w_value)
return 1
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,7 +15,7 @@
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rtyper.annlowlevel import llhelper
from rpython.rlib import rawrefcount, jit
-from rpython.rlib.debug import fatalerror
+from rpython.rlib.debug import ll_assert, fatalerror
#________________________________________________________
@@ -243,6 +243,11 @@
py_obj = rawrefcount.from_obj(PyObject, w_obj)
if not py_obj:
py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal)
+ #
+ # Try to crash here, instead of randomly, if we don't keep w_obj alive
+ ll_assert(py_obj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY,
+ "Bug in cpyext: The W_Root object was garbage-collected "
+ "while being converted to PyObject.")
return py_obj
else:
return lltype.nullptr(PyObject.TO)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -13,7 +13,7 @@
ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
getbufferproc, ssizessizeobjargproc)
-from pypy.module.cpyext.pyobject import make_ref, decref, from_ref
+from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj
from pypy.module.cpyext.pyerrors import PyErr_Occurred
from pypy.module.cpyext.memoryobject import fill_Py_buffer
from pypy.module.cpyext.state import State
@@ -90,20 +90,21 @@
args_w = space.fixedview(w_args)
return generic_cpy_call(space, func_binary, w_self, args_w[0])
+def _get_ob_type(space, w_obj):
+ # please ensure that w_obj stays alive
+ ob_type = as_pyobj(space, space.type(w_obj))
+ return rffi.cast(PyTypeObjectPtr, ob_type)
+
def wrap_binaryfunc_l(space, w_self, w_args, func):
func_binary = rffi.cast(binaryfunc, func)
check_num_args(space, w_args, 1)
args_w = space.fixedview(w_args)
- ref = make_ref(space, w_self)
- decref(space, ref)
return generic_cpy_call(space, func_binary, w_self, args_w[0])
def wrap_binaryfunc_r(space, w_self, w_args, func):
func_binary = rffi.cast(binaryfunc, func)
check_num_args(space, w_args, 1)
args_w = space.fixedview(w_args)
- ref = make_ref(space, w_self)
- decref(space, ref)
return generic_cpy_call(space, func_binary, args_w[0], w_self)
def wrap_ternaryfunc(space, w_self, w_args, func):
@@ -121,8 +122,6 @@
func_ternary = rffi.cast(ternaryfunc, func)
check_num_argsv(space, w_args, 1, 2)
args_w = space.fixedview(w_args)
- ref = make_ref(space, w_self)
- decref(space, ref)
arg3 = space.w_None
if len(args_w) > 1:
arg3 = args_w[1]
@@ -314,12 +313,10 @@
def wrap_getreadbuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
- py_obj = make_ref(space, w_self)
- py_type = py_obj.c_ob_type
+ py_type = _get_ob_type(space, w_self)
rbp = rffi.cast(rffi.VOIDP, 0)
if py_type.c_tp_as_buffer:
rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
- decref(space, py_obj)
with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
index = rffi.cast(Py_ssize_t, 0)
size = generic_cpy_call(space, func_target, w_self, index, ptr)
@@ -332,9 +329,7 @@
def wrap_getwritebuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
- py_obj = make_ref(space, w_self)
- py_type = py_obj.c_ob_type
- decref(space, py_obj)
+ py_type = _get_ob_type(space, w_self)
rbp = rffi.cast(rffi.VOIDP, 0)
if py_type.c_tp_as_buffer:
rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
@@ -350,12 +345,10 @@
def wrap_getbuffer(space, w_self, w_args, func):
func_target = rffi.cast(getbufferproc, func)
- py_obj = make_ref(space, w_self)
- py_type = py_obj.c_ob_type
+ py_type = _get_ob_type(space, w_self)
rbp = rffi.cast(rffi.VOIDP, 0)
if py_type.c_tp_as_buffer:
rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
- decref(space, py_obj)
with lltype.scoped_alloc(Py_buffer) as pybuf:
_flags = 0
if space.len_w(w_args) > 0:
diff --git a/pypy/module/cpyext/test/test_typeobject.py
b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -337,12 +337,8 @@
PyObject* name = PyBytes_FromString("mymodule");
PyObject *obj = PyType_Type.tp_alloc(&PyType_Type, 0);
PyHeapTypeObject *type = (PyHeapTypeObject*)obj;
- if ((type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0)
- {
- PyErr_SetString(PyExc_ValueError,
- "Py_TPFLAGS_HEAPTYPE not set");
- return NULL;
- }
+ /* this is issue #2434: logic from pybind11 */
+ type->ht_type.tp_flags |= Py_TPFLAGS_HEAPTYPE;
type->ht_type.tp_name = ((PyTypeObject*)args)->tp_name;
PyType_Ready(&type->ht_type);
ret = PyObject_SetAttrString((PyObject*)&type->ht_type,
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py
b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -307,6 +307,18 @@
assert module.utf8('xyz') == b'xyz'
assert module.utf8('café') == 'café'.encode('utf-8')
+ def test_ready(self):
+ module = self.import_extension('foo', [
+ ("unsafe_len", "METH_O",
+ """
+ Py_ssize_t size = PyUnicode_GET_LENGTH(args);
+ return PyLong_FromSsize_t(size);
+ """)])
+ assert module.unsafe_len(u"abc") == 3
+ assert module.unsafe_len(u"café") == 4
+ assert module.unsafe_len(u'aАbБcСdД') == 8
+ assert module.unsafe_len(u"café\U0001F4A9") == 5
+
class TestUnicode(BaseApiTest):
def test_unicodeobject(self, space):
@@ -863,30 +875,31 @@
assert space.unwrap(PyUnicode_Substring(space, w_str, 4, 3)) == u""
def test_Ready(self, space):
- w_str = space.wrap(u'abc') # ASCII
- py_str = as_pyobj(space, w_str)
+ def as_py_uni(val):
+ py_obj = new_empty_unicode(space, len(val))
+ set_wbuffer(py_obj, rffi.unicode2wcharp(val))
+ return py_obj
+
+ py_str = as_py_uni(u'abc') # ASCII
assert get_kind(py_str) == 0
- _PyUnicode_Ready(space, w_str)
+ _PyUnicode_Ready(space, py_str)
assert get_kind(py_str) == 1
assert get_ascii(py_str) == 1
- w_str = space.wrap(u'café') # latin1
- py_str = as_pyobj(space, w_str)
+ py_str = as_py_uni(u'café') # latin1
assert get_kind(py_str) == 0
- _PyUnicode_Ready(space, w_str)
+ _PyUnicode_Ready(space, py_str)
assert get_kind(py_str) == 1
assert get_ascii(py_str) == 0
- w_str =
space.wrap(u'Росси́я') # UCS2
- py_str = as_pyobj(space, w_str)
+ py_str =
as_py_uni(u'Росси́я') # UCS2
assert get_kind(py_str) == 0
- _PyUnicode_Ready(space, w_str)
+ _PyUnicode_Ready(space, py_str)
assert get_kind(py_str) == 2
assert get_ascii(py_str) == 0
- w_str = space.wrap(u'***\U0001f4a9***') # UCS4
- py_str = as_pyobj(space, w_str)
+ py_str = as_py_uni(u'***\U0001f4a9***') # UCS4
assert get_kind(py_str) == 0
- _PyUnicode_Ready(space, w_str)
+ _PyUnicode_Ready(space, py_str)
assert get_kind(py_str) == 4
assert get_ascii(py_str) == 0
diff --git a/pypy/module/cpyext/unicodeobject.py
b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -68,8 +68,10 @@
def unicode_attach(space, py_obj, w_obj, w_userdata=None):
"Fills a newly allocated PyUnicodeObject with a unicode string"
- set_wsize(py_obj, len(space.unicode_w(w_obj)))
+ value = space.unicode_w(w_obj)
+ set_wsize(py_obj, len(value))
set_wbuffer(py_obj, lltype.nullptr(rffi.CWCHARP.TO))
+ _readify(space, py_obj, value)
def unicode_realize(space, py_obj):
"""
@@ -265,8 +267,11 @@
assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
py_obj = as_pyobj(space, w_obj)
assert get_kind(py_obj) == WCHAR_KIND
+ return _readify(space, py_obj, w_obj._value)
+
+def _readify(space, py_obj, value):
maxchar = 0
- for c in w_obj._value:
+ for c in value:
if ord(c) > maxchar:
maxchar = ord(c)
if maxchar > MAX_UNICODE:
@@ -275,7 +280,7 @@
maxchar)
if maxchar < 256:
ucs1_data = rffi.str2charp(unicode_encode_latin_1(
- w_obj._value, len(w_obj._value), errors='strict'))
+ value, len(value), errors='strict'))
set_data(py_obj, cts.cast('void*', ucs1_data))
set_kind(py_obj, _1BYTE_KIND)
set_len(py_obj, get_wsize(py_obj))
@@ -290,7 +295,7 @@
elif maxchar < 65536:
# XXX: assumes that sizeof(wchar_t) == 4
ucs2_str = unicode_encode_utf_16_helper(
- w_obj._value, len(w_obj._value), errors='strict',
+ value, len(value), errors='strict',
byteorder=runicode.BYTEORDER)
ucs2_data = cts.cast('Py_UCS2 *', rffi.str2charp(ucs2_str))
set_data(py_obj, cts.cast('void*', ucs2_data))
@@ -302,9 +307,8 @@
# XXX: assumes that sizeof(wchar_t) == 4
if not get_wbuffer(py_obj):
# Copy unicode buffer
- u = w_obj._value
- set_wbuffer(py_obj, rffi.unicode2wcharp(u))
- set_wsize(py_obj, len(u))
+ set_wbuffer(py_obj, rffi.unicode2wcharp(value))
+ set_wsize(py_obj, len(value))
ucs4_data = get_wbuffer(py_obj)
set_data(py_obj, cts.cast('void*', ucs4_data))
set_len(py_obj, get_wsize(py_obj))
@@ -332,9 +336,7 @@
w_res = utf_32_decode(space, value, w_final=space.w_False)
else:
raise oefmt(space.w_SystemError, "invalid kind")
- w_ret = space.unpackiterable(w_res)[0]
- _PyUnicode_Ready(space, w_ret)
- return w_ret
+ return space.unpackiterable(w_res)[0]
@cts.decl("Py_UNICODE * PyUnicode_AsUnicodeAndSize(PyObject *unicode,
Py_ssize_t *size)")
def PyUnicode_AsUnicodeAndSize(space, ref, psize):
diff --git a/pypy/module/imp/test/test_import.py
b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -601,25 +601,15 @@
import pkg.a, imp
imp.reload(pkg.a)
- def test_reload_builtin(self):
- import sys, imp
- oldpath = sys.path
- try:
- del sys.settrace
- except AttributeError:
- pass
-
- imp.reload(sys)
-
- assert sys.path is oldpath
- assert 'settrace' not in dir(sys) # at least on CPython 3.5.2
-
def test_reload_builtin_doesnt_clear(self):
import imp
import sys
sys.foobar = "baz"
- imp.reload(sys)
- assert sys.foobar == "baz"
+ try:
+ imp.reload(sys)
+ assert sys.foobar == "baz"
+ finally:
+ del sys.foobar
def test_reimport_builtin_simple_case_1(self):
import sys, time
@@ -637,18 +627,18 @@
def test_reimport_builtin(self):
import imp, sys, time
- oldpath = sys.path
- time.tzname = "<test_reimport_builtin removed this>"
+ old_sleep = time.sleep
+ time.sleep = "<test_reimport_builtin removed this>"
del sys.modules['time']
import time as time1
assert sys.modules['time'] is time1
- assert time.tzname == "<test_reimport_builtin removed this>"
+ assert time.sleep == "<test_reimport_builtin removed this>"
- imp.reload(time1) # don't leave a broken time.tzname behind
+ imp.reload(time1) # don't leave a broken time.sleep behind
import time
- assert time.tzname != "<test_reimport_builtin removed this>"
+ assert time.sleep is old_sleep
def test_reload_infinite(self):
import infinite_reload
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
@@ -37,7 +37,7 @@
v = cffi.__version__.replace('+', '')
p = os.path.join(parent, 'doc', 'source', 'installation.rst')
content = open(p).read()
- assert ("/cffi-%s.tar.gz" % v) in content
+ assert (" package version %s:" % v) in content
def test_setup_version():
parent = os.path.dirname(os.path.dirname(cffi.__file__))
diff --git a/pypy/module/unicodedata/test/test_hyp.py
b/pypy/module/unicodedata/test/test_hyp.py
--- a/pypy/module/unicodedata/test/test_hyp.py
+++ b/pypy/module/unicodedata/test/test_hyp.py
@@ -40,6 +40,7 @@
@pytest.mark.parametrize('NF1, NF2, NF3', compositions)
@example(s=u'---\uafb8\u11a7---') # issue 2289
+@example(s=u'\ufacf')
@settings(max_examples=1000)
@given(s=st.text())
def test_composition(s, space, NF1, NF2, NF3):
diff --git a/pypy/objspace/std/test/test_bytesobject.py
b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -1,4 +1,7 @@
# coding: utf-8
+
+from pypy.interpreter.error import OperationError
+
class TestW_BytesObject:
def teardown_method(self, method):
@@ -96,6 +99,78 @@
w_res = space.call_function(space.w_bytes, space.wrap([42]))
assert space.str_w(w_res) == '*'
+
+try:
+ from hypothesis import given, strategies
+except ImportError:
+ pass
+else:
+ @given(u=strategies.binary(),
+ start=strategies.integers(min_value=0, max_value=10),
+ len1=strategies.integers(min_value=-1, max_value=10))
+ def test_hypo_index_find(u, start, len1, space):
+ if start + len1 < 0:
+ return # skip this case
+ v = u[start : start + len1]
+ w_u = space.wrap(u)
+ w_v = space.wrap(v)
+ expected = u.find(v, start, start + len1)
+ try:
+ w_index = space.call_method(w_u, 'index', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ assert expected == -1
+ else:
+ assert space.int_w(w_index) == expected >= 0
+
+ w_index = space.call_method(w_u, 'find', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert space.int_w(w_index) == expected
+
+ rexpected = u.rfind(v, start, start + len1)
+ try:
+ w_index = space.call_method(w_u, 'rindex', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ assert rexpected == -1
+ else:
+ assert space.int_w(w_index) == rexpected >= 0
+
+ w_index = space.call_method(w_u, 'rfind', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert space.int_w(w_index) == rexpected
+
+ expected = u.startswith(v, start)
+ w_res = space.call_method(w_u, 'startswith', w_v,
+ space.newint(start))
+ assert w_res is space.newbool(expected)
+
+ expected = u.startswith(v, start, start + len1)
+ w_res = space.call_method(w_u, 'startswith', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert w_res is space.newbool(expected)
+
+ expected = u.endswith(v, start)
+ w_res = space.call_method(w_u, 'endswith', w_v,
+ space.newint(start))
+ assert w_res is space.newbool(expected)
+
+ expected = u.endswith(v, start, start + len1)
+ w_res = space.call_method(w_u, 'endswith', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert w_res is space.newbool(expected)
+
+
class AppTestBytesObject:
def setup_class(cls):
diff --git a/pypy/objspace/std/test/test_unicodeobject.py
b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -1,6 +1,7 @@
# -*- encoding: utf-8 -*-
import py
import sys
+from pypy.interpreter.error import OperationError
class TestUnicodeObject:
@@ -38,6 +39,55 @@
space.raises_w(space.w_UnicodeEncodeError, space.text_w, w_uni)
+try:
+ from hypothesis import given, strategies
+except ImportError:
+ pass
+else:
+ @given(u=strategies.text(),
+ start=strategies.integers(min_value=0, max_value=10),
+ len1=strategies.integers(min_value=-1, max_value=10))
+ def test_hypo_index_find(u, start, len1, space):
+ if start + len1 < 0:
+ return # skip this case
+ v = u[start : start + len1]
+ w_u = space.wrap(u)
+ w_v = space.wrap(v)
+ expected = u.find(v, start, start + len1)
+ try:
+ w_index = space.call_method(w_u, 'index', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ assert expected == -1
+ else:
+ assert space.int_w(w_index) == expected >= 0
+
+ w_index = space.call_method(w_u, 'find', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert space.int_w(w_index) == expected
+
+ rexpected = u.rfind(v, start, start + len1)
+ try:
+ w_index = space.call_method(w_u, 'rindex', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ assert rexpected == -1
+ else:
+ assert space.int_w(w_index) == rexpected >= 0
+
+ w_index = space.call_method(w_u, 'rfind', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert space.int_w(w_index) == rexpected
+
+
class AppTestUnicodeStringStdOnly:
def test_compares(self):
assert type('a') != type(b'a')
diff --git a/pypy/objspace/std/unicodeobject.py
b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -748,17 +748,6 @@
interpreted as in slice notation.
"""
- def decode():
- """S.decode(encoding=None, errors='strict') -> string or unicode
-
- Decode S using the codec registered for encoding. encoding defaults
- to the default encoding. errors may be given to set a different error
- handling scheme. Default is 'strict' meaning that encoding errors raise
- a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'
- as well as any other name registered with codecs.register_error that is
- able to handle UnicodeDecodeErrors.
- """
-
def encode():
"""S.encode(encoding=None, errors='strict') -> string or unicode
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -57,9 +57,6 @@
if not ok:
py.test.skip("cannot runappdirect test: "
"module %r required" % (modname,))
- else:
- if '__pypy__' in value:
- py.test.skip("no module __pypy__ on top of CPython")
continue
if info is None:
py.test.skip("cannot runappdirect this test on top of CPython")
diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst
--- a/rpython/doc/jit/optimizer.rst
+++ b/rpython/doc/jit/optimizer.rst
@@ -42,10 +42,9 @@
There are better ways to compute the sum from ``[0..100]``, but it gives a
better intuition on how
traces are constructed than ``sum(range(101))``.
Note that the trace syntax is the one used in the test suite. It is also very
-similar to traces printed at runtime by PYPYLOG_. The first line gives the
input variables, the
-second line is a ``label`` operation, the last one is the backwards ``jump``
operation.
-
-.. _PYPYLOG: logging.html
+similar to traces printed at runtime by :doc:`PYPYLOG <../logging>`. The first
+line gives the input variables, the second line is a ``label`` operation, the
+last one is the backwards ``jump`` operation.
These instructions mentioned earlier are special:
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py
b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -305,6 +305,10 @@
# Transform into INT_ADD. The following guard will be killed
# by optimize_GUARD_NO_OVERFLOW; if we see instead an
# optimize_GUARD_OVERFLOW, then InvalidLoop.
+
+ # NB: this case also takes care of int_add_ovf with 0 as on of the
+ # arguments: the result will be bounded, and then the optimization
+ # for int_add with 0 as argument will remove the op.
op = self.replace_op_with(op, rop.INT_ADD)
return self.emit(op)
@@ -325,6 +329,7 @@
return None
resbound = b0.sub_bound(b1)
if resbound.bounded():
+ # this case takes care of int_sub_ovf(x, 0) as well
op = self.replace_op_with(op, rop.INT_SUB)
return self.emit(op)
@@ -342,6 +347,7 @@
b2 = self.getintbound(op.getarg(1))
resbound = b1.mul_bound(b2)
if resbound.bounded():
+ # this case also takes care of multiplication with 0 and 1
op = self.replace_op_with(op, rop.INT_MUL)
return self.emit(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -1962,6 +1962,55 @@
"""
self.optimize_loop(ops, expected)
+ ops = """
+ [i0]
+ i1 = int_mul_ovf(0, i0)
+ guard_no_overflow() []
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(0)
+ """
+ self.optimize_loop(ops, expected)
+
+ ops = """
+ [i0]
+ i1 = int_mul_ovf(i0, 0)
+ guard_no_overflow() []
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(0)
+ """
+ self.optimize_loop(ops, expected)
+
+ ops = """
+ [i0]
+ i1 = int_mul_ovf(1, i0)
+ guard_no_overflow() []
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(i0)
+ """
+ self.optimize_loop(ops, expected)
+
+ ops = """
+ [i0]
+ i1 = int_mul_ovf(i0, 1)
+ guard_no_overflow() []
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(i0)
+ """
+ self.optimize_loop(ops, expected)
+
+
def test_fold_constant_partial_ops_float(self):
ops = """
[f0]
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -162,7 +162,7 @@
IP_RECVRETOPTS IP_RETOPTS IP_TOS IP_TTL
MSG_BTAG MSG_ETAG MSG_CTRUNC MSG_DONTROUTE MSG_DONTWAIT MSG_EOR MSG_OOB
-MSG_PEEK MSG_TRUNC MSG_WAITALL
+MSG_PEEK MSG_TRUNC MSG_WAITALL MSG_ERRQUEUE
NI_DGRAM NI_MAXHOST NI_MAXSERV NI_NAMEREQD NI_NOFQDN NI_NUMERICHOST
NI_NUMERICSERV
@@ -535,7 +535,7 @@
int cmsg_status;
struct iovec iov;
struct recvmsg_info* retinfo;
- int error_flag; // variable to be set in case of special errors.
+ int error_flag = 0; // variable to be set in case of special
errors.
int cmsgdatalen = 0;
// variables that are set to 1, if the message charp has been
allocated
@@ -709,6 +709,7 @@
free(retinfo);
}
}
+ if (error_flag==0) error_flag = -1;
return error_flag;
err_closefds:
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1074,7 +1074,7 @@
if address is not None:
address.unlock()
- if _c.geterrno() == _c.EINTR:
+ if (_c.geterrno() == _c.EINTR) or (_c.geterrno() == 11):
raise last_error()
if (reply == -10000):
raise RSocketError("Invalid message size")
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -877,32 +877,31 @@
ch = ord(s[pos])
pos += 1
ch2 = 0
- if 0xD800 <= ch < 0xDC00:
- if not allow_surrogates:
- ru, rs, pos = errorhandler(errors, public_encoding_name,
- 'surrogates not allowed',
- s, pos-1, pos)
- if rs is not None:
- # py3k only
- if len(rs) % 4 != 0:
- errorhandler('strict', public_encoding_name,
- 'surrogates not allowed',
- s, pos-1, pos)
- result.append(rs)
- continue
- for ch in ru:
- if ord(ch) < 0xD800:
- _STORECHAR32(result, ord(ch), byteorder)
- else:
- errorhandler('strict', public_encoding_name,
- 'surrogates not allowed',
- s, pos-1, pos)
+ if not allow_surrogates and 0xD800 <= ch < 0xE000:
+ ru, rs, pos = errorhandler(errors, public_encoding_name,
+ 'surrogates not allowed',
+ s, pos-1, pos)
+ if rs is not None:
+ # py3k only
+ if len(rs) % 4 != 0:
+ errorhandler('strict', public_encoding_name,
+ 'surrogates not allowed',
+ s, pos-1, pos)
+ result.append(rs)
continue
- elif MAXUNICODE < 65536 and pos < size:
- ch2 = ord(s[pos])
- if 0xDC00 <= ch2 < 0xE000:
- ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
- pos += 1
+ for ch in ru:
+ if ord(ch) < 0xD800:
+ _STORECHAR32(result, ord(ch), byteorder)
+ else:
+ errorhandler('strict', public_encoding_name,
+ 'surrogates not allowed',
+ s, pos-1, pos)
+ continue
+ if 0xD800 <= ch < 0xDC00 and MAXUNICODE < 65536 and pos < size:
+ ch2 = ord(s[pos])
+ if 0xDC00 <= ch2 < 0xE000:
+ ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
+ pos += 1
_STORECHAR32(result, ch, byteorder)
return result.build()
diff --git a/rpython/rlib/test/test_runicode.py
b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -2,6 +2,7 @@
import py
import sys, random
+import struct
from rpython.rlib import runicode
from hypothesis import given, settings, strategies
@@ -266,11 +267,12 @@
assert replace_with(u'rep', None) == '\x00<\x00r\x00e\x00p\x00>'
assert replace_with(None, '\xca\xfe') == '\x00<\xca\xfe\x00>'
- def test_utf32_surrogates(self):
+ @py.test.mark.parametrize('unich',[u"\ud800", u"\udc80"])
+ def test_utf32_surrogates(self, unich):
assert runicode.unicode_encode_utf_32_be(
- u"\ud800", 1, None) == '\x00\x00\xd8\x00'
+ unich, 1, None) == struct.pack('>i', ord(unich))
py.test.raises(UnicodeEncodeError, runicode.unicode_encode_utf_32_be,
- u"\ud800", 1, None, allow_surrogates=False)
+ unich, 1, None, allow_surrogates=False)
def replace_with(ru, rs):
def errorhandler(errors, enc, msg, u, startingpos, endingpos):
if errors == 'strict':
@@ -278,7 +280,7 @@
endingpos, msg)
return ru, rs, endingpos
return runicode.unicode_encode_utf_32_be(
- u"<\ud800>", 3, None,
+ u"<%s>" % unich, 3, None,
errorhandler, allow_surrogates=False)
assert replace_with(u'rep', None) == u'<rep>'.encode('utf-32-be')
assert replace_with(None, '\xca\xfe\xca\xfe') ==
'\x00\x00\x00<\xca\xfe\xca\xfe\x00\x00\x00>'
@@ -432,7 +434,7 @@
assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'ignore',
final=True) == (u'aaaabbbb', len(seq) + 8))
assert (self.decoder(seq, len(seq), 'custom', final=True,
- errorhandler=self.custom_replace) ==
+ errorhandler=self.custom_replace) ==
(FOO * len(seq), len(seq)))
assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'custom',
final=True, errorhandler=self.custom_replace) ==
@@ -628,7 +630,7 @@
msg='invalid continuation byte')
assert self.decoder(seq, len(seq), 'replace', final=True
) == (res, len(seq))
- assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8,
+ assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8,
'replace', final=True) ==
(u'aaaa' + res + u'bbbb', len(seq) + 8))
res = res.replace(FFFD, u'')
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
