Author: Ronan Lamy <ronan.l...@gmail.com>
Branch: buffer-cleanup
Changeset: r89987:062ca5e63c28
Date: 2017-02-05 20:42 +0000
http://bitbucket.org/pypy/pypy/changeset/062ca5e63c28/
Log: hg merge py3.5
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -439,3 +439,8 @@
return (type(self), (self.default_factory,), None, None,
iter(self.items()))
+
+try:
+ from _pypy_collections import OrderedDict
+except ImportError:
+ pass
diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_collections.py
@@ -0,0 +1,70 @@
+from __pypy__ import reversed_dict, move_to_end
+from _operator import eq as _eq
+from reprlib import recursive_repr as _recursive_repr
+
+class OrderedDict(dict):
+ '''Dictionary that remembers insertion order.
+
+ In PyPy all dicts are ordered anyway. This is mostly useful as a
+ placeholder to mean "this dict must be ordered even on CPython".
+
+ Known difference: iterating over an OrderedDict which is being
+ concurrently modified raises RuntimeError in PyPy. In CPython
+ instead we get some behavior that appears reasonable in some
+ cases but is nonsensical in other cases. This is officially
+ forbidden by the CPython docs, so we forbid it explicitly for now.
+ '''
+
+ def __reversed__(self):
+ return reversed_dict(self)
+
+ def popitem(self, last=True):
+ '''od.popitem() -> (k, v), return and remove a (key, value) pair.
+ Pairs are returned in LIFO order if last is true or FIFO order if
false.
+
+ '''
+ if last:
+ return dict.popitem(self)
+ else:
+ it = dict.__iter__(self)
+ try:
+ k = it.next()
+ except StopIteration:
+ raise KeyError('dictionary is empty')
+ return (k, self.pop(k))
+
+ def move_to_end(self, key, last=True):
+ '''Move an existing element to the end (or beginning if last==False).
+
+ Raises KeyError if the element does not exist.
+ When last=True, acts like a fast version of self[key]=self.pop(key).
+
+ '''
+ return move_to_end(self, key, last)
+
+ @_recursive_repr()
+ def __repr__(self):
+ 'od.__repr__() <==> repr(od)'
+ if not self:
+ return '%s()' % (self.__class__.__name__,)
+ return '%s(%r)' % (self.__class__.__name__, list(self.items()))
+
+ def __reduce__(self):
+ 'Return state information for pickling'
+ inst_dict = vars(self).copy()
+ return self.__class__, (), inst_dict or None, None, iter(self.items())
+
+ def copy(self):
+ 'od.copy() -> a shallow copy of od'
+ return self.__class__(self)
+
+ def __eq__(self, other):
+ '''od.__eq__(y) <==> od==y. Comparison to another OD is
order-sensitive
+ while comparison to a regular mapping is order-insensitive.
+
+ '''
+ if isinstance(other, OrderedDict):
+ return dict.__eq__(self, other) and all(map(_eq, self, other))
+ return dict.__eq__(self, other)
+
+ __ne__ = object.__ne__
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -93,6 +93,7 @@
# ctypes backend: attach these constants to the instance
self.NULL = self.cast(self.BVoidP, 0)
self.CData, self.CType = backend._get_types()
+ self.buffer = backend.buffer
def cdef(self, csource, override=False, packed=False):
"""Parse the given C source. This registers all declared functions,
@@ -316,18 +317,18 @@
"""
return self._backend.unpack(cdata, length)
- def buffer(self, cdata, size=-1):
- """Return a read-write buffer object that references the raw C data
- pointed to by the given 'cdata'. The 'cdata' must be a pointer or
- an array. Can be passed to functions expecting a buffer, or directly
- manipulated with:
-
- buf[:] get a copy of it in a regular string, or
- buf[idx] as a single character
- buf[:] = ...
- buf[idx] = ... change the content
- """
- return self._backend.buffer(cdata, size)
+ #def buffer(self, cdata, size=-1):
+ # """Return a read-write buffer object that references the raw C data
+ # pointed to by the given 'cdata'. The 'cdata' must be a pointer or
+ # an array. Can be passed to functions expecting a buffer, or directly
+ # manipulated with:
+ #
+ # buf[:] get a copy of it in a regular string, or
+ # buf[idx] as a single character
+ # buf[:] = ...
+ # buf[idx] = ... change the content
+ # """
+ # note that 'buffer' is a type, set on this instance by __init__
def from_buffer(self, python_buffer):
"""Return a <cdata 'char[]'> that points to the data of the
@@ -593,11 +594,15 @@
ensure('extra_link_args', '/MANIFEST')
def set_source(self, module_name, source, source_extension='.c', **kwds):
+ import os
if hasattr(self, '_assigned_source'):
raise ValueError("set_source() cannot be called several times "
"per ffi object")
if not isinstance(module_name, basestring):
raise TypeError("'module_name' must be a string")
+ if os.sep in module_name or (os.altsep and os.altsep in module_name):
+ raise ValueError("'module_name' must not contain '/': use a dotted
"
+ "name to make a 'package.module' location")
self._assigned_source = (str(module_name), source,
source_extension, kwds)
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -190,7 +190,7 @@
from GC issues: if the program "forgets" an unfinished greenlet, it will
always be collected at the next garbage collection.
-.. _documentation of the greenlets: http://packages.python.org/greenlet/
+.. _documentation of the greenlets: https://greenlet.readthedocs.io/
Unimplemented features
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -78,6 +78,7 @@
'add_memory_pressure' : 'interp_magic.add_memory_pressure',
'newdict' : 'interp_dict.newdict',
'reversed_dict' : 'interp_dict.reversed_dict',
+ 'move_to_end' : 'interp_dict.move_to_end',
'strategy' : 'interp_magic.strategy', # dict,set,list
'set_debug' : 'interp_magic.set_debug',
'locals_to_fast' : 'interp_magic.locals_to_fast',
diff --git a/pypy/module/__pypy__/interp_dict.py
b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -44,3 +44,18 @@
if not isinstance(w_obj, W_DictMultiObject):
raise OperationError(space.w_TypeError, space.w_None)
return w_obj.nondescr_reversed_dict(space)
+
+@unwrap_spec(last=bool)
+def move_to_end(space, w_obj, w_key, last=True):
+ """Move the key in a dictionary object into the first or last position.
+
+ This is a __pypy__ function instead of being simply done by calling
+ dict.move_to_end(), for CPython compatibility: dictionaries are only
+ ordered on PyPy. You should use the collections.OrderedDict class for
+ cases where ordering is important. That class implements the
+ move_to_end() method by calling __pypy__.move_to_end().
+ """
+ from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+ if not isinstance(w_obj, W_DictMultiObject):
+ raise OperationError(space.w_TypeError, space.w_None)
+ return w_obj.nondescr_move_to_end(space, w_key, last)
diff --git a/pypy/module/_cffi_backend/__init__.py
b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -50,7 +50,7 @@
'string': 'func.string',
'unpack': 'func.unpack',
- 'buffer': 'cbuffer.buffer',
+ 'buffer': 'cbuffer.MiniBuffer',
'memmove': 'func.memmove',
'get_errno': 'cerrno.get_errno',
diff --git a/pypy/module/_cffi_backend/cbuffer.py
b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -75,18 +75,8 @@
self.buffer.setslice(start, value.as_str())
-MiniBuffer.typedef = TypeDef(
- "_cffi_backend.buffer",
- __len__ = interp2app(MiniBuffer.descr_len),
- __getitem__ = interp2app(MiniBuffer.descr_getitem),
- __setitem__ = interp2app(MiniBuffer.descr_setitem),
- __weakref__ = make_weakref_descr(MiniBuffer),
- )
-MiniBuffer.typedef.acceptable_as_base_class = False
-
-
@unwrap_spec(w_cdata=cdataobj.W_CData, size=int)
-def buffer(space, w_cdata, size=-1):
+def MiniBuffer___new__(space, w_subtype, w_cdata, size=-1):
ctype = w_cdata.ctype
if isinstance(ctype, ctypeptr.W_CTypePointer):
if size < 0:
@@ -107,3 +97,24 @@
"don't know the size pointed to by '%s'", ctype.name)
ptr = w_cdata.unsafe_escaping_ptr() # w_cdata kept alive by MiniBuffer()
return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
+
+MiniBuffer.typedef = TypeDef(
+ "_cffi_backend.buffer",
+ __new__ = interp2app(MiniBuffer___new__),
+ __len__ = interp2app(MiniBuffer.descr_len),
+ __getitem__ = interp2app(MiniBuffer.descr_getitem),
+ __setitem__ = interp2app(MiniBuffer.descr_setitem),
+ __weakref__ = make_weakref_descr(MiniBuffer),
+ __doc__ = """ffi.buffer(cdata[, byte_size]):
+Return a read-write buffer object that references the raw C data
+pointed to by the given 'cdata'. The 'cdata' must be a pointer or an
+array. Can be passed to functions expecting a buffer, or directly
+manipulated with:
+
+ buf[:] get a copy of it in a regular string, or
+ buf[idx] as a single character
+ buf[:] = ...
+ buf[idx] = ... change the content
+""",
+ )
+MiniBuffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ffi_obj.py
b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -265,22 +265,6 @@
return self.space.wrap(align)
- @unwrap_spec(w_cdata=W_CData, size=int)
- def descr_buffer(self, w_cdata, size=-1):
- """\
-Return a read-write buffer object that references the raw C data
-ointed to by the given 'cdata'. The 'cdata' must be a pointer or an
-array. Can be passed to functions expecting a buffer, or directly
-manipulated with:
-
- buf[:] get a copy of it in a regular string, or
- buf[idx] as a single character
- buf[:] = ...
- buf[idx] = ... change the content"""
- #
- return cbuffer.buffer(self.space, w_cdata, size)
-
-
@unwrap_spec(w_name=WrappedDefault(None),
w_error=WrappedDefault(None),
w_onerror=WrappedDefault(None))
@@ -751,6 +735,9 @@
return space.appexec([], """():
return type('error', (Exception,), {'__module__': 'ffi'})""")
+def make_buffer(space):
+ return space.gettypefor(cbuffer.MiniBuffer)
+
_extras = get_dict_rtld_constants()
if sys.platform == 'win32':
_extras['getwinerror'] = interp2app(W_FFIObject.descr_getwinerror)
@@ -770,7 +757,7 @@
cls=W_FFIObject),
addressof = interp2app(W_FFIObject.descr_addressof),
alignof = interp2app(W_FFIObject.descr_alignof),
- buffer = interp2app(W_FFIObject.descr_buffer),
+ buffer = ClassAttr(make_buffer),
callback = interp2app(W_FFIObject.descr_callback),
cast = interp2app(W_FFIObject.descr_cast),
def_extern = interp2app(W_FFIObject.descr_def_extern),
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2288,6 +2288,7 @@
buf = buffer(c)
assert repr(buf).startswith('<_cffi_backend.buffer object at 0x')
assert bytes(buf) == b"hi there\x00"
+ assert type(buf) is buffer
if sys.version_info < (3,):
assert str(buf) == "hi there\x00"
assert unicode(buf) == u+"hi there\x00"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py
b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -256,6 +256,10 @@
ffi = _cffi1_backend.FFI()
a = ffi.new("signed char[]", [5, 6, 7])
assert ffi.buffer(a)[:] == b'\x05\x06\x07'
+ assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+ assert type(ffi.buffer(a)) is ffi.buffer
+ assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+ assert type(ffi.buffer(a)) is ffi.buffer
def test_ffi_from_buffer(self):
import _cffi_backend as _cffi1_backend
diff --git a/pypy/module/_collections/__init__.py
b/pypy/module/_collections/__init__.py
--- a/pypy/module/_collections/__init__.py
+++ b/pypy/module/_collections/__init__.py
@@ -25,3 +25,15 @@
space = self.space
space.getattr(self, space.wrap('defaultdict')) # force importing
space.delattr(self, space.wrap('__missing__'))
+
+ def startup(self, space):
+ # OrderedDict is normally present, but in some cases the line
+ # "from __pypy__ import reversed_dict, move_to_end" from
+ # _pypy_collections.py raises
+ space.appexec([space.wrap(self)], """(mod):
+ try:
+ from _pypy_collections import OrderedDict
+ mod.OrderedDict = OrderedDict
+ except ImportError:
+ pass
+ """)
diff --git a/pypy/module/_collections/test/test_ordereddict.py
b/pypy/module/_collections/test/test_ordereddict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_collections/test/test_ordereddict.py
@@ -0,0 +1,8 @@
+
+class AppTestBasic:
+ spaceconfig = dict(usemodules=['_collections'])
+
+ def test_ordereddict_present(self):
+ from _collections import OrderedDict
+ assert issubclass(OrderedDict, dict)
+ assert hasattr(OrderedDict, 'move_to_end')
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -40,6 +40,7 @@
import pypy.module.cpyext.pyerrors
import pypy.module.cpyext.typeobject
import pypy.module.cpyext.object
+import pypy.module.cpyext.buffer
import pypy.module.cpyext.bytesobject
import pypy.module.cpyext.bytearrayobject
import pypy.module.cpyext.tupleobject
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -126,7 +126,7 @@
Py_TPFLAGS_HEAPTYPE
Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
Py_CLEANUP_SUPPORTED
-PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE PyBUF_SIMPLE
""".split()
for name in constant_names:
setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -571,6 +571,7 @@
'_Py_BuildValue_SizeT', '_Py_VaBuildValue_SizeT',
'PyErr_Format', 'PyErr_NewException', 'PyErr_NewExceptionWithDoc',
+ 'PyErr_WarnFormat',
'PySys_WriteStdout', 'PySys_WriteStderr',
'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction',
@@ -612,6 +613,9 @@
'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag',
'Py_IgnoreEnvironmentFlag',
'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag',
'Py_NoUserSiteDirectory',
'_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag',
'_Py_PackageContext',
+
+ 'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree',
+ 'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free',
]
TYPES = {}
FORWARD_DECLS = []
@@ -1074,7 +1078,8 @@
struct PyPyAPI {
%(members)s
} _pypyAPI;
- RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
+ RPY_EXTERN struct PyPyAPI* pypyAPI;
+ struct PyPyAPI* pypyAPI = &_pypyAPI;
""" % dict(members=structmembers)
global_objects = []
@@ -1318,6 +1323,7 @@
source_dir / "bytesobject.c",
source_dir / "complexobject.c",
source_dir / "import.c",
+ source_dir / "_warnings.c",
]
def build_eci(code, use_micronumpy=False, translating=False):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/buffer.py
@@ -0,0 +1,72 @@
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.interpreter.error import oefmt
+from pypy.module.cpyext.api import (
+ cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP,
+ generic_cpy_call,
+ PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES, PyBUF_SIMPLE)
+from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
+
+@cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
+def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
+ """Returns a pointer to a read-only memory location usable as
+ character-based input. The obj argument must support the single-segment
+ character buffer interface. On success, returns 0, sets buffer to the
+ memory location and size to the buffer length. Returns -1 and sets a
+ TypeError on error.
+ """
+ pto = obj.c_ob_type
+ pb = pto.c_tp_as_buffer
+ if not (pb and pb.c_bf_getbuffer):
+ raise oefmt(space.w_TypeError,
+ "expected an object with the buffer interface")
+ with lltype.scoped_alloc(Py_buffer) as view:
+ ret = generic_cpy_call(
+ space, pb.c_bf_getbuffer,
+ obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
+ if rffi.cast(lltype.Signed, ret) == -1:
+ return -1
+
+ bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
+ sizep[0] = view.c_len
+
+ if pb.c_bf_releasebuffer:
+ generic_cpy_call(space, pb.c_bf_releasebuffer,
+ obj, view)
+ Py_DecRef(space, view.c_obj)
+ return 0
+
+
+@cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
+ lltype.Signed, lltype.Signed], rffi.INT, error=-1)
+def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
+ """
+ Fills in a buffer-info structure correctly for an exporter that can only
+ share a contiguous chunk of memory of "unsigned bytes" of the given
+ length. Returns 0 on success and -1 (with raising an error) on error.
+ """
+ flags = rffi.cast(lltype.Signed, flags)
+ if flags & PyBUF_WRITABLE and readonly:
+ raise oefmt(space.w_ValueError, "Object is not writable")
+ view.c_buf = buf
+ view.c_len = length
+ view.c_obj = obj
+ if obj:
+ Py_IncRef(space, obj)
+ view.c_itemsize = 1
+ rffi.setintfield(view, 'c_readonly', readonly)
+ rffi.setintfield(view, 'c_ndim', 1)
+ view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+ if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+ view.c_format = rffi.str2charp("B")
+ view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+ if (flags & PyBUF_ND) == PyBUF_ND:
+ view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+ view.c_shape[0] = view.c_len
+ view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+ if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+ view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+ view.c_strides[0] = view.c_itemsize
+ view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+ view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
+ return 0
diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/genobject.py
@@ -0,0 +1,7 @@
+from pypy.interpreter.generator import GeneratorIterator, Coroutine
+from pypy.module.cpyext.api import build_type_checkers
+
+
+PyGen_Check, PyGen_CheckExact = build_type_checkers("Gen", GeneratorIterator)
+
+_, PyCoro_CheckExact = build_type_checkers("Coro", Coroutine)
diff --git a/pypy/module/cpyext/include/pymem.h
b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -7,17 +7,22 @@
extern "C" {
#endif
-#define PyMem_MALLOC(n) malloc((n) ? (n) : 1)
-#define PyMem_REALLOC(p, n) realloc((p), (n) ? (n) : 1)
-#define PyMem_FREE free
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
+PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
+#endif
-PyAPI_FUNC(void *) PyMem_Malloc(size_t);
-#define PyMem_Free PyMem_FREE
-#define PyMem_Realloc PyMem_REALLOC
+PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
+PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_Free(void *ptr);
-#define PyMem_RawMalloc PyMem_Malloc
-#define PyMem_RawFree PyMem_Free
-#define PyMem_RawRealloc PyMem_Realloc
+#define PyMem_MALLOC(n) PyMem_Malloc(n)
+#define PyMem_REALLOC(p, n) PyMem_Realloc(p, n)
+#define PyMem_FREE(p) PyMem_Free(p)
+
/*
* Type-oriented memory interface
diff --git a/pypy/module/cpyext/include/warnings.h
b/pypy/module/cpyext/include/warnings.h
--- a/pypy/module/cpyext/include/warnings.h
+++ b/pypy/module/cpyext/include/warnings.h
@@ -6,6 +6,9 @@
#define PyErr_WarnPy3k(msg, stacklevel) 0
+PyAPI_FUNC(int) PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
+ const char *format, ...);
+
#ifdef __cplusplus
}
#endif
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,13 +1,11 @@
from rpython.rtyper.lltypesystem import rffi, lltype
from pypy.module.cpyext.api import (
- cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
- PyVarObject, Py_buffer, size_t, slot_function, cts,
- PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
+ cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t,
+ PyVarObject, size_t, slot_function, cts,
Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
- Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
+ Py_GE, CONST_STRING, FILEP, fwrite)
from pypy.module.cpyext.pyobject import (
- PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
- get_typedescr, _Py_NewReference)
+ PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef, get_typedescr)
from pypy.module.cpyext.typeobject import PyTypeObjectPtr
from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
from pypy.objspace.std.typeobject import W_TypeObject
@@ -16,10 +14,6 @@
import pypy.module.__builtin__.operation as operation
-# from include/object.h
-PyBUF_SIMPLE = 0x0000
-PyBUF_WRITABLE = 0x0001
-
@cpython_api([size_t], rffi.VOIDP)
def PyObject_Malloc(space, size):
# returns non-zero-initialized memory, like CPython
@@ -444,36 +438,6 @@
is active then NULL is returned but PyErr_Occurred() will return false."""
return space.call_function(space.builtin.get('dir'), w_o)
-@cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
-def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
- """Returns a pointer to a read-only memory location usable as
- character-based input. The obj argument must support the single-segment
- character buffer interface. On success, returns 0, sets buffer to the
- memory location and size to the buffer length. Returns -1 and sets a
- TypeError on error.
- """
- pto = obj.c_ob_type
-
- pb = pto.c_tp_as_buffer
- if not (pb and pb.c_bf_getbuffer):
- raise oefmt(space.w_TypeError,
- "expected an object with the buffer interface")
- with lltype.scoped_alloc(Py_buffer) as view:
- ret = generic_cpy_call(
- space, pb.c_bf_getbuffer,
- obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
- if rffi.cast(lltype.Signed, ret) == -1:
- return -1
-
- bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
- sizep[0] = view.c_len
-
- if pb.c_bf_releasebuffer:
- generic_cpy_call(space, pb.c_bf_releasebuffer,
- obj, view)
- Py_DecRef(space, view.c_obj)
- return 0
-
# Also in include/object.h
Py_PRINT_RAW = 1 # No string quotes etc.
@@ -493,41 +457,3 @@
with rffi.scoped_nonmovingbuffer(data) as buf:
fwrite(buf, 1, count, fp)
return 0
-
-
-PyBUF_WRITABLE = 0x0001 # Copied from object.h
-
-@cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
- lltype.Signed, lltype.Signed], rffi.INT, error=-1)
-def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
- """
- Fills in a buffer-info structure correctly for an exporter that can only
- share a contiguous chunk of memory of "unsigned bytes" of the given
- length. Returns 0 on success and -1 (with raising an error) on error.
- """
- flags = rffi.cast(lltype.Signed, flags)
- if flags & PyBUF_WRITABLE and readonly:
- raise oefmt(space.w_ValueError, "Object is not writable")
- view.c_buf = buf
- view.c_len = length
- view.c_obj = obj
- if obj:
- Py_IncRef(space, obj)
- view.c_itemsize = 1
- rffi.setintfield(view, 'c_readonly', readonly)
- rffi.setintfield(view, 'c_ndim', 1)
- view.c_format = lltype.nullptr(rffi.CCHARP.TO)
- if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
- view.c_format = rffi.str2charp("B")
- view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
- if (flags & PyBUF_ND) == PyBUF_ND:
- view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
- view.c_shape[0] = view.c_len
- view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
- if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
- view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
- view.c_strides[0] = view.c_itemsize
- view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
- view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-
- return 0
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -1,6 +1,7 @@
from pypy.module.cpyext.api import (
cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct)
from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref, from_ref
+from pypy.interpreter.error import OperationError
from rpython.rtyper.lltypesystem import rffi, lltype
from rpython.rlib import rthread
from rpython.rlib.objectmodel import we_are_translated
@@ -317,3 +318,16 @@
be held. The thread state must have been reset with a previous call to
PyThreadState_Clear()."""
+@cpython_api([], lltype.Void)
+def PyOS_AfterFork(space):
+ """Function to update some internal state after a process fork; this
should be
+ called in the new process if the Python interpreter will continue to be
used.
+ If a new executable is loaded into the new process, this function does not
need
+ to be called."""
+ if not space.config.translation.thread:
+ return
+ from pypy.module.thread import os_thread
+ try:
+ os_thread.reinit_threads(space)
+ except OperationError as e:
+ e.write_unraisable(space, "PyOS_AfterFork()")
diff --git a/pypy/module/cpyext/src/_warnings.c
b/pypy/module/cpyext/src/_warnings.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/_warnings.c
@@ -0,0 +1,25 @@
+#include <Python.h>
+
+int
+PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
+ const char *format, ...)
+{
+ int ret;
+ PyObject *message;
+ va_list vargs;
+
+#ifdef HAVE_STDARG_PROTOTYPES
+ va_start(vargs, format);
+#else
+ va_start(vargs);
+#endif
+ message = PyUnicode_FromFormatV(format, vargs);
+ if (message != NULL) {
+ ret = PyErr_WarnEx(category, PyUnicode_AsUTF8(message), stack_level);
+ Py_DECREF(message);
+ }
+ else
+ ret = -1;
+ va_end(vargs);
+ return ret;
+}
diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c
--- a/pypy/module/cpyext/src/pymem.c
+++ b/pypy/module/cpyext/src/pymem.c
@@ -1,6 +1,86 @@
#include <Python.h>
-void * PyMem_Malloc(size_t n)
+void *
+PyMem_RawMalloc(size_t size)
{
- return malloc((n) ? (n) : 1);
+ /*
+ * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
+ * Most python internals blindly use a signed Py_ssize_t to track
+ * things without checking for overflows or negatives.
+ * As size_t is unsigned, checking for size < 0 is not required.
+ */
+ if (size > (size_t)PY_SSIZE_T_MAX)
+ return NULL;
+ if (size == 0)
+ size = 1;
+ return malloc(size);
}
+
+void *
+PyMem_RawCalloc(size_t nelem, size_t elsize)
+{
+ /* see PyMem_RawMalloc() */
+ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+ return NULL;
+ /* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL
+ for calloc(0, 0), which would be treated as an error. Some platforms
+ would return a pointer with no memory behind it, which would break
+ pymalloc. To solve these problems, allocate an extra byte. */
+ if (nelem == 0 || elsize == 0) {
+ nelem = 1;
+ elsize = 1;
+ }
+ return calloc(nelem, elsize);
+}
+
+void*
+PyMem_RawRealloc(void *ptr, size_t size)
+{
+ /* see PyMem_RawMalloc() */
+ if (size > (size_t)PY_SSIZE_T_MAX)
+ return NULL;
+ if (size == 0)
+ size = 1;
+ return realloc(ptr, size);
+}
+
+void PyMem_RawFree(void *ptr)
+{
+ free(ptr);
+}
+
+
+/* the PyMem_Xxx functions are the same as PyMem_RawXxx in PyPy, for now */
+void *PyMem_Malloc(size_t size)
+{
+ if (size > (size_t)PY_SSIZE_T_MAX)
+ return NULL;
+ if (size == 0)
+ size = 1;
+ return malloc(size);
+}
+
+void *PyMem_Calloc(size_t nelem, size_t elsize)
+{
+ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+ return NULL;
+ if (nelem == 0 || elsize == 0) {
+ nelem = 1;
+ elsize = 1;
+ }
+ return calloc(nelem, elsize);
+}
+
+void* PyMem_Realloc(void *ptr, size_t size)
+{
+ if (size > (size_t)PY_SSIZE_T_MAX)
+ return NULL;
+ if (size == 0)
+ size = 1;
+ return realloc(ptr, size);
+}
+
+void PyMem_Free(void *ptr)
+{
+ free(ptr);
+}
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -405,14 +405,6 @@
(sys.getfilesystemencoding())."""
raise NotImplementedError
-@cpython_api([PyObject, Py_ssize_t, rffi.CCHARP, ], rffi.INT_real, error=-1)
-def PyErr_WarnFormat(space, category, stack_level, format, ):
- """Function similar to PyErr_WarnEx(), but use
- PyUnicode_FromFormat() to format the warning message. format is
- an ASCII-encoded string.
- """
- raise NotImplementedError
-
@cpython_api([rffi.INT_real], rffi.INT_real, error=-1)
def PySignal_SetWakeupFd(space, fd):
@@ -668,17 +660,6 @@
extension modules."""
raise NotImplementedError
-@cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyGen_Check(space, ob):
- """Return true if ob is a generator object; ob must not be NULL."""
- raise NotImplementedError
-
-@cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyGen_CheckExact(space, ob):
- """Return true if ob's type is PyGen_Type is a generator object; ob must
not
- be NULL."""
- raise NotImplementedError
-
@cpython_api([PyFrameObject], PyObject)
def PyGen_New(space, frame):
"""Create and return a new generator object based on the frame object. A
@@ -1487,14 +1468,6 @@
one of the strings '<stdin>' or '???'."""
raise NotImplementedError
-@cpython_api([], lltype.Void)
-def PyOS_AfterFork(space):
- """Function to update some internal state after a process fork; this
should be
- called in the new process if the Python interpreter will continue to be
used.
- If a new executable is loaded into the new process, this function does not
need
- to be called."""
- raise NotImplementedError
-
@cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
def PyOS_CheckStack(space):
"""Return true when the interpreter runs out of stack space. This is a
reliable
diff --git a/pypy/module/cpyext/test/test_genobject.py
b/pypy/module/cpyext/test/test_genobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -0,0 +1,27 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.genobject import PyGen_Check, PyGen_CheckExact
+from pypy.module.cpyext.genobject import PyCoro_CheckExact
+
+
+class TestGenObject(BaseApiTest):
+ def test_genobject(self, space):
+ w_geniter = space.appexec([], """():
+ def f():
+ yield 42
+ return f()
+ """)
+ assert PyGen_Check(space, w_geniter)
+ assert PyGen_CheckExact(space, w_geniter)
+ assert not PyCoro_CheckExact(space, w_geniter)
+ assert not PyGen_Check(space, space.wrap(2))
+ assert not PyGen_CheckExact(space, space.wrap("b"))
+ assert not PyCoro_CheckExact(space, space.wrap([]))
+
+ w_coroutine = space.appexec([], """():
+ async def f():
+ pass
+ return f()
+ """)
+ assert not PyGen_Check(space, w_coroutine)
+ assert not PyGen_CheckExact(space, w_coroutine)
+ assert PyCoro_CheckExact(space, w_coroutine)
diff --git a/pypy/module/cpyext/test/test_pyerrors.py
b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -428,3 +428,18 @@
assert orig_exc_info == reset_sys_exc_info
assert new_exc_info == (new_exc.__class__, new_exc, None)
assert new_exc_info == new_sys_exc_info
+
+ def test_PyErr_WarnFormat(self):
+ import warnings
+
+ module = self.import_extension('foo', [
+ ("test", "METH_NOARGS",
+ '''
+ PyErr_WarnFormat(PyExc_UserWarning, 1, "foo %d bar", 42);
+ Py_RETURN_NONE;
+ '''),
+ ])
+ with warnings.catch_warnings(record=True) as l:
+ module.test()
+ assert len(l) == 1
+ assert "foo 42 bar" in str(l[0])
diff --git a/pypy/module/cpyext/test/test_pymem.py
b/pypy/module/cpyext/test/test_pymem.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_pymem.py
@@ -0,0 +1,34 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class AppTestPyMem(AppTestCpythonExtensionBase):
+ def test_pymem_alloc(self):
+ module = self.import_extension('foo', [
+ ("test", "METH_NOARGS",
+ """
+ int *a, *b;
+ a = PyMem_RawCalloc(4, 50);
+ if (a[49] != 0) {
+ PyErr_SetString(PyExc_ValueError, "1");
+ return NULL;
+ }
+ a[49] = 123456;
+ b = PyMem_RawRealloc(a, 2000);
+ b[499] = 789123;
+ PyMem_RawFree(b);
+
+ a = PyMem_Calloc(4, 50);
+ if (a[49] != 0) {
+ PyErr_SetString(PyExc_ValueError, "2");
+ return NULL;
+ }
+ a[49] = 123456;
+ b = PyMem_Realloc(a, 2000);
+ b[499] = 789123;
+ PyMem_Free(b);
+
+ Py_RETURN_NONE;
+ """),
+ ])
+ res = module.test()
+ assert res is None
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -548,14 +548,14 @@
@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
rffi.INT_real, error=-1)
def bytes_getbuffer(space, w_str, view, flags):
from pypy.module.cpyext.bytesobject import PyBytes_AsString
- from pypy.module.cpyext.object import PyBuffer_FillInfo
+ from pypy.module.cpyext.buffer import PyBuffer_FillInfo
c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str))
return PyBuffer_FillInfo(space, view, w_str, c_buf,
space.len_w(w_str), 1, flags)
@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
rffi.INT_real, error=-1)
def bf_getbuffer(space, w_obj, view, flags):
- from pypy.module.cpyext.object import PyBuffer_FillInfo
+ from pypy.module.cpyext.buffer import PyBuffer_FillInfo
buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
return PyBuffer_FillInfo(space, view, w_obj, c_buf,
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1131,6 +1131,7 @@
b = ffi.buffer(a)
except NotImplementedError as e:
py.test.skip(str(e))
+ assert type(b) is ffi.buffer
content = b[:]
assert len(content) == len(b) == 2
if sys.byteorder == 'little':
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -234,6 +234,7 @@
a = ffi.new("signed char[]", [5, 6, 7])
assert ffi.buffer(a)[:] == b'\x05\x06\x07'
assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+ assert type(ffi.buffer(a)) is ffi.buffer
def test_ffi_from_buffer():
import array
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -37,6 +37,11 @@
['-Werror'])
return recompiler._verify(ffi, module_name, source, *args, **kwds)
+def test_set_source_no_slashes():
+ ffi = FFI()
+ py.test.raises(ValueError, ffi.set_source, "abc/def", None)
+ py.test.raises(ValueError, ffi.set_source, "abc/def", "C code")
+
def test_type_table_func():
check_type_table("double sin(double);",
diff --git a/pypy/objspace/std/dictmultiobject.py
b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -232,6 +232,37 @@
w_keys = self.w_keys()
return space.call_method(w_keys, '__reversed__')
+ def nondescr_move_to_end(self, space, w_key, last_flag):
+ """Not exposed directly to app-level, but via __pypy__.move_to_end().
+ """
+ strategy = self.get_strategy()
+ if strategy.has_move_to_end:
+ strategy.move_to_end(self, w_key, last_flag)
+ else:
+ # fall-back
+ w_value = self.getitem(w_key)
+ if w_value is None:
+ space.raise_key_error(w_key)
+ else:
+ self.delitem(w_key)
+ if last_flag:
+ self.setitem(w_key, w_value)
+ else:
+ # *very slow* fall-back
+ keys_w = []
+ values_w = []
+ iteratorimplementation = self.iteritems()
+ while True:
+ w_k, w_v = iteratorimplementation.next_item()
+ if w_k is None:
+ break
+ keys_w.append(w_k)
+ values_w.append(w_v)
+ self.clear()
+ self.setitem(w_key, w_value)
+ for i in range(len(keys_w)):
+ self.setitem(keys_w[i], values_w[i])
+
def descr_clear(self, space):
"""D.clear() -> None. Remove all items from D."""
self.clear()
@@ -499,7 +530,9 @@
raise NotImplementedError
has_iterreversed = False
- # no 'getiterreversed': no default implementation available
+ has_move_to_end = False
+ # no 'getiterreversed' and no 'move_to_end': no default
+ # implementation available
def rev_update1_dict_dict(self, w_dict, w_updatedict):
iteritems = self.iteritems(w_dict)
@@ -783,6 +816,9 @@
dictimpl.iterreversed = iterreversed
dictimpl.has_iterreversed = True
+ if hasattr(dictimpl, 'move_to_end'):
+ dictimpl.has_move_to_end = True
+
@jit.look_inside_iff(lambda self, w_dict, w_updatedict:
w_dict_unrolling_heuristic(w_dict))
def rev_update1_dict_dict(self, w_dict, w_updatedict):
@@ -962,6 +998,15 @@
def getiterreversed(self, w_dict):
return objectmodel.reversed_dict(self.unerase(w_dict.dstorage))
+ def move_to_end(self, w_dict, w_key, last_flag):
+ if self.is_correct_type(w_key):
+ d = self.unerase(w_dict.dstorage)
+ key = self.unwrap(w_key)
+ objectmodel.move_to_end(d, key, last_flag)
+ else:
+ self.switch_to_object_strategy(w_dict)
+ w_dict.nondescr_move_to_end(w_dict.space, w_key, last_flag)
+
def prepare_update(self, w_dict, num_extra):
objectmodel.prepare_dict_update(self.unerase(w_dict.dstorage),
num_extra)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py
b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -270,6 +270,29 @@
del d[key]
raises(RuntimeError, next, it)
+ def test_move_to_end(self):
+ import __pypy__
+ raises(KeyError, __pypy__.move_to_end, {}, 'foo')
+ raises(KeyError, __pypy__.move_to_end, {}, 'foo', last=True)
+ raises(KeyError, __pypy__.move_to_end, {}, 'foo', last=False)
+ def kwdict(**k):
+ return k
+ for last in [False, True]:
+ for d, key in [({1: 2, 3: 4, 5: 6}, 3),
+ ({"a": 5, "b": 2, "c": 6}, "b"),
+ (kwdict(d=7, e=8, f=9), "e")]:
+ other_keys = [k for k in d if k != key]
+ __pypy__.move_to_end(d, key, last=last)
+ if not self.on_pypy:
+ # when running tests on CPython, the underlying
+ # dicts are not ordered. We don't get here if
+ # we're running tests on PyPy or with -A.
+ assert set(d.keys()) == set(other_keys + [key])
+ elif last:
+ assert list(d) == other_keys + [key]
+ else:
+ assert list(d) == [key] + other_keys
+
def test_keys(self):
d = {1: 2, 3: 4}
kys = list(d.keys())
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -14,7 +14,7 @@
SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
SomeFloat, SomeIterator, SomePBC, SomeNone, SomeTypeOf, s_ImpossibleValue,
s_Bool, s_None, s_Int, unionof, add_knowntypedata,
- SomeWeakRef, SomeUnicodeString, SomeByteArray)
+ SomeWeakRef, SomeUnicodeString, SomeByteArray, SomeOrderedDict)
from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
from rpython.annotator.binaryop import _clone ## XXX where to put this?
from rpython.annotator.binaryop import _dict_can_only_throw_keyerror
@@ -575,6 +575,13 @@
pair(self, s_key).delitem()
method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
+class __extend__(SomeOrderedDict):
+
+ def method_move_to_end(self, s_key, s_last):
+ assert s_Bool.contains(s_last)
+ pair(self, s_key).delitem()
+ method_move_to_end.can_only_throw = _dict_can_only_throw_keyerror
+
@op.contains.register(SomeString)
@op.contains.register(SomeUnicodeString)
def contains_String(annotator, string, char):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,24 @@
return
d.delitem_with_hash(key, h)
+def _untranslated_move_to_end(d, key, last):
+ "NOT_RPYTHON"
+ value = d.pop(key)
+ if last:
+ d[key] = value
+ else:
+ items = d.items()
+ d.clear()
+ d[key] = value
+ d.update(items)
+
+@specialize.call_location()
+def move_to_end(d, key, last=True):
+ if not we_are_translated():
+ _untranslated_move_to_end(d, key, last)
+ return
+ d.move_to_end(key, last)
+
# ____________________________________________________________
def import_from_mixin(M, special_methods=['__init__', '__del__']):
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -602,7 +602,7 @@
if rposix.HAVE_FSTATAT:
from rpython.rlib.rposix import AT_FDCWD, AT_SYMLINK_NOFOLLOW
- c_fstatat = rffi.llexternal('fstatat',
+ c_fstatat = rffi.llexternal('fstatat64' if _LINUX else 'fstatat',
[rffi.INT, rffi.CCHARP, STAT_STRUCT, rffi.INT], rffi.INT,
compilation_info=compilation_info,
save_err=rffi.RFFI_SAVE_ERRNO, macro=True)
diff --git a/rpython/rlib/test/test_objectmodel.py
b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
- fetch_translated_config, try_inline)
+ fetch_translated_config, try_inline, move_to_end)
from rpython.translator.translator import TranslationContext, graphof
from rpython.rtyper.test.tool import BaseRtypingTest
from rpython.rtyper.test.test_llinterp import interpret
@@ -679,6 +679,16 @@
assert f(29) == 0
interpret(f, [27])
+def test_rordereddict_move_to_end():
+ d = OrderedDict()
+ d['key1'] = 'val1'
+ d['key2'] = 'val2'
+ d['key3'] = 'val3'
+ move_to_end(d, 'key1')
+ assert d.items() == [('key2', 'val2'), ('key3', 'val3'), ('key1', 'val1')]
+ move_to_end(d, 'key1', last=False)
+ assert d.items() == [('key1', 'val1'), ('key2', 'val2'), ('key3', 'val3')]
+
def test_import_from_mixin():
class M: # old-style
def f(self):
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py
b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,15 @@
hop.exception_is_here()
hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
+ def rtype_method_move_to_end(self, hop):
+ v_dict, v_key, v_last = hop.inputargs(
+ self, self.key_repr, lltype.Bool)
+ if not self.custom_eq_hash:
+ hop.has_implicit_exception(KeyError) # record that we know about
it
+ hop.exception_is_here()
+ hop.gendirectcall(ll_dict_move_to_end, v_dict, v_key, v_last)
+
+
class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
def rtype_getitem((r_dict, r_key), hop):
@@ -542,16 +551,18 @@
ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
assert False
-def ll_call_delete_by_entry_index(d, hash, i):
+def ll_call_delete_by_entry_index(d, hash, i, replace_with):
+ # only called from _ll_dict_del, whose @jit.look_inside_iff
+ # condition should control when we get inside here with the jit
fun = d.lookup_function_no & FUNC_MASK
if fun == FUNC_BYTE:
- ll_dict_delete_by_entry_index(d, hash, i, TYPE_BYTE)
+ ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_BYTE)
elif fun == FUNC_SHORT:
- ll_dict_delete_by_entry_index(d, hash, i, TYPE_SHORT)
+ ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_SHORT)
elif IS_64BIT and fun == FUNC_INT:
- ll_dict_delete_by_entry_index(d, hash, i, TYPE_INT)
+ ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_INT)
elif fun == FUNC_LONG:
- ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
+ ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_LONG)
else:
# can't be still FUNC_MUST_REINDEX here
ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
@@ -805,13 +816,12 @@
ll_dict_delitem_with_hash(d, key, d.keyhash(key))
def ll_dict_delitem_with_hash(d, key, hash):
- index = d.lookup_function(d, key, hash, FLAG_DELETE)
+ index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
if index < 0:
raise KeyError
- _ll_dict_del(d, index)
+ _ll_dict_del(d, hash, index)
-@jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
-def _ll_dict_del(d, index):
+def _ll_dict_del_entry(d, index):
d.entries.mark_deleted(index)
d.num_live_items -= 1
# clear the key and the value if they are GC pointers
@@ -823,6 +833,11 @@
if ENTRIES.must_clear_value:
entry.value = lltype.nullptr(ENTRY.value.TO)
+@jit.look_inside_iff(lambda d, h, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, hash, index):
+ ll_call_delete_by_entry_index(d, hash, index, DELETED)
+ _ll_dict_del_entry(d, index)
+
if d.num_live_items == 0:
# Dict is now empty. Reset these fields.
d.num_ever_used_items = 0
@@ -963,7 +978,6 @@
FLAG_LOOKUP = 0
FLAG_STORE = 1
-FLAG_DELETE = 2
@specialize.memo()
def _ll_ptr_to_array_of(T):
@@ -985,8 +999,6 @@
if index >= VALID_OFFSET:
checkingkey = entries[index - VALID_OFFSET].key
if direct_compare and checkingkey == key:
- if store_flag == FLAG_DELETE:
- indexes[i] = rffi.cast(T, DELETED)
return index - VALID_OFFSET # found the entry
if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
# correct hash, maybe the key is e.g. a different pointer to
@@ -1000,8 +1012,6 @@
# the compare did major nasty stuff to the dict: start over
return ll_dict_lookup(d, key, hash, store_flag, T)
if found:
- if store_flag == FLAG_DELETE:
- indexes[i] = rffi.cast(T, DELETED)
return index - VALID_OFFSET
deletedslot = -1
elif index == DELETED:
@@ -1030,8 +1040,6 @@
elif index >= VALID_OFFSET:
checkingkey = entries[index - VALID_OFFSET].key
if direct_compare and checkingkey == key:
- if store_flag == FLAG_DELETE:
- indexes[i] = rffi.cast(T, DELETED)
return index - VALID_OFFSET # found the entry
if d.keyeq is not None and entries.hash(index - VALID_OFFSET) ==
hash:
# correct hash, maybe the key is e.g. a different pointer to
@@ -1044,8 +1052,6 @@
# the compare did major nasty stuff to the dict: start
over
return ll_dict_lookup(d, key, hash, store_flag, T)
if found:
- if store_flag == FLAG_DELETE:
- indexes[i] = rffi.cast(T, DELETED)
return index - VALID_OFFSET
elif deletedslot == -1:
deletedslot = intmask(i)
@@ -1066,7 +1072,11 @@
perturb >>= PERTURB_SHIFT
indexes[i] = rffi.cast(T, index + VALID_OFFSET)
-def ll_dict_delete_by_entry_index(d, hash, locate_index, T):
+# the following function is only called from _ll_dict_del, whose
+# @jit.look_inside_iff condition should control when we get inside
+# here with the jit
+@jit.unroll_safe
+def ll_dict_delete_by_entry_index(d, hash, locate_index, replace_with, T):
# Another simplified version of ll_dict_lookup() which locates a
# hashtable entry with the given 'index' stored in it, and deletes it.
# This *should* be safe against evil user-level __eq__/__hash__
@@ -1083,7 +1093,7 @@
i = (i << 2) + i + perturb + 1
i = i & mask
perturb >>= PERTURB_SHIFT
- indexes[i] = rffi.cast(T, DELETED)
+ indexes[i] = rffi.cast(T, replace_with)
# ____________________________________________________________
#
@@ -1253,18 +1263,8 @@
if hasattr(DICT, 'fnkeyhash'):
newdict.fnkeyhash = dict.fnkeyhash
- i = 0
- while i < newdict.num_ever_used_items:
- d_entry = newdict.entries[i]
- entry = dict.entries[i]
- ENTRY = lltype.typeOf(newdict.entries).TO.OF
- d_entry.key = entry.key
- if hasattr(ENTRY, 'f_valid'):
- d_entry.f_valid = entry.f_valid
- d_entry.value = entry.value
- if hasattr(ENTRY, 'f_hash'):
- d_entry.f_hash = entry.f_hash
- i += 1
+ rgc.ll_arraycopy(dict.entries, newdict.entries, 0, 0,
+ newdict.num_ever_used_items)
ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
return newdict
@@ -1390,8 +1390,6 @@
break
dic.num_ever_used_items -= 1
- # we must remove the precise entry in the hashtable that points to 'i'
- ll_call_delete_by_entry_index(dic, entries.hash(i), i)
return i
def ll_dict_popitem(ELEM, dic):
@@ -1400,21 +1398,139 @@
r = lltype.malloc(ELEM.TO)
r.item0 = recast(ELEM.TO.item0, entry.key)
r.item1 = recast(ELEM.TO.item1, entry.value)
- _ll_dict_del(dic, i)
+ _ll_dict_del(dic, dic.entries.hash(i), i)
return r
def ll_dict_pop(dic, key):
- index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+ hash = dic.keyhash(key)
+ index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
if index < 0:
raise KeyError
value = dic.entries[index].value
- _ll_dict_del(dic, index)
+ _ll_dict_del(dic, hash, index)
return value
def ll_dict_pop_default(dic, key, dfl):
- index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+ hash = dic.keyhash(key)
+ index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
if index < 0:
return dfl
value = dic.entries[index].value
- _ll_dict_del(dic, index)
+ _ll_dict_del(dic, hash, index)
return value
+
+def ll_dict_move_to_end(d, key, last):
+ if last:
+ ll_dict_move_to_last(d, key)
+ else:
+ ll_dict_move_to_first(d, key)
+
+def ll_dict_move_to_last(d, key):
+ hash = d.keyhash(key)
+ old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+ if old_index < 0:
+ raise KeyError
+
+ if old_index == d.num_ever_used_items - 1:
+ return
+
+ # remove the entry at the old position
+ old_entry = d.entries[old_index]
+ key = old_entry.key
+ value = old_entry.value
+ _ll_dict_del_entry(d, old_index)
+
+ # note a corner case: it is possible that 'replace_with' is just too
+ # large to fit in the type T used so far for the index. But in that
+ # case, the list 'd.entries' is full, and the following call to
+ # _ll_dict_setitem_lookup_done() will necessarily reindex the dict.
+ # So in that case, this value of 'replace_with' should be ignored.
+ ll_call_delete_by_entry_index(d, hash, old_index,
+ replace_with = VALID_OFFSET + d.num_ever_used_items)
+ _ll_dict_setitem_lookup_done(d, key, value, hash, -1)
+
+def ll_dict_move_to_first(d, key):
+ # In this function, we might do a bit more than the strict minimum
+ # of walks over parts of the array, trying to keep the code at least
+ # semi-reasonable, while the goal is still amortized constant-time
+ # over many calls.
+
+ # Call ll_dict_remove_deleted_items() first if there are too many
+ # deleted items. Not a perfect solution, because lookup_function()
+ # might do random things with the dict and create many new deleted
+ # items. Still, should be fine, because nothing crucially depends
+ # on this: the goal is to avoid the dictionary's list growing
+ # forever.
+ if d.num_live_items < len(d.entries) // 2 - 16:
+ ll_dict_remove_deleted_items(d)
+
+ hash = d.keyhash(key)
+ old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+ if old_index <= 0:
+ if old_index < 0:
+ raise KeyError
+ else:
+ return
+
+ # the goal of the following is to set 'idst' to the number of
+ # deleted entries at the beginning, ensuring 'idst > 0'
+ must_reindex = False
+ if d.entries.valid(0):
+ # the first entry is valid, so we need to make room before.
+ new_allocated = _overallocate_entries_len(d.num_ever_used_items)
+ idst = ((new_allocated - d.num_ever_used_items) * 3) // 4
+ ll_assert(idst > 0, "overallocate did not do enough")
+ newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+ rgc.ll_arraycopy(d.entries, newitems, 0, idst, d.num_ever_used_items)
+ d.entries = newitems
+ i = 0
+ while i < idst:
+ d.entries.mark_deleted(i)
+ i += 1
+ d.num_ever_used_items += idst
+ old_index += idst
+ must_reindex = True
+ idst -= 1
+ else:
+ idst = d.lookup_function_no >> FUNC_SHIFT
+ # All entries in range(0, idst) are deleted. Check if more are
+ while not d.entries.valid(idst):
+ idst += 1
+ if idst == old_index:
+ d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+ (old_index << FUNC_SHIFT))
+ return
+ idst -= 1
+ d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+ (idst << FUNC_SHIFT))
+
+ # remove the entry at the old position
+ ll_assert(d.entries.valid(old_index),
+ "ll_dict_move_to_first: lost old_index")
+ ENTRY = lltype.typeOf(d.entries).TO.OF
+ old_entry = d.entries[old_index]
+ key = old_entry.key
+ value = old_entry.value
+ if hasattr(ENTRY, 'f_hash'):
+ ll_assert(old_entry.f_hash == hash,
+ "ll_dict_move_to_first: bad hash")
+ _ll_dict_del_entry(d, old_index)
+
+ # put the entry at its new position
+ ll_assert(not d.entries.valid(idst),
+ "ll_dict_move_to_first: overwriting idst")
+ new_entry = d.entries[idst]
+ new_entry.key = key
+ new_entry.value = value
+ if hasattr(ENTRY, 'f_hash'):
+ new_entry.f_hash = hash
+ if hasattr(ENTRY, 'f_valid'):
+ new_entry.f_valid = True
+ d.num_live_items += 1
+
+ # fix the index
+ if must_reindex:
+ ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+ else:
+ ll_call_delete_by_entry_index(d, hash, old_index,
+ replace_with = VALID_OFFSET + idst)
diff --git a/rpython/rtyper/test/test_rdict.py
b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1188,6 +1188,12 @@
assert not self.ll_contains(self.l_dict, ll_key)
self.removed_keys.append(key)
+ def move_to_end(self, key, last=True):
+ "For test_rordereddict"
+
+ def move_to_first(self, key):
+ self.move_to_end(key, last=False)
+
def copydict(self):
self.l_dict = self.ll_copy(self.l_dict)
assert self.ll_len(self.l_dict) == len(self.reference)
@@ -1250,6 +1256,15 @@
return builds(Action,
just('delitem'), tuples(sampled_from(self.space.reference)))
+ def st_move_to_end(self):
+ return builds(Action,
+ just('move_to_end'), tuples(sampled_from(self.space.reference)))
+
+ def st_move_to_first(self):
+ return builds(Action,
+ just('move_to_first'),
+ tuples(sampled_from(self.space.reference)))
+
def steps(self):
if not self.space:
return builds(Action, just('setup'), tuples(st_keys, st_values))
@@ -1258,7 +1273,8 @@
if self.space.reference:
return (
self.st_setitem() | sampled_from(global_actions) |
- self.st_updateitem() | self.st_delitem())
+ self.st_updateitem() | self.st_delitem() |
+ self.st_move_to_end() | self.st_move_to_first())
else:
return (self.st_setitem() | sampled_from(global_actions))
diff --git a/rpython/rtyper/test/test_rordereddict.py
b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -1,7 +1,8 @@
import py
+import random
from collections import OrderedDict
-from hypothesis import settings
+from hypothesis import settings, given, strategies
from hypothesis.stateful import run_state_machine_as_test
from rpython.rtyper.lltypesystem import lltype, rffi
@@ -145,14 +146,18 @@
ll_d = rordereddict.ll_newdict(DICT)
rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
rordereddict.ll_dict_setitem(ll_d, llstr("j"), 2)
- ITER = rordereddict.get_ll_dictiter(lltype.Ptr(DICT))
+ assert [hlstr(entry.key) for entry in self._ll_iter(ll_d)] == ["k",
"j"]
+
+ def _ll_iter(self, ll_d):
+ ITER = rordereddict.get_ll_dictiter(lltype.typeOf(ll_d))
ll_iter = rordereddict.ll_dictiter(ITER, ll_d)
ll_dictnext = rordereddict._ll_dictnext
- num = ll_dictnext(ll_iter)
- assert hlstr(ll_d.entries[num].key) == "k"
- num = ll_dictnext(ll_iter)
- assert hlstr(ll_d.entries[num].key) == "j"
- py.test.raises(StopIteration, ll_dictnext, ll_iter)
+ while True:
+ try:
+ num = ll_dictnext(ll_iter)
+ except StopIteration:
+ break
+ yield ll_d.entries[num]
def test_popitem(self):
DICT = self._get_str_dict()
@@ -337,6 +342,31 @@
num_nonfrees += (got > 0)
assert d.resize_counter <= idx.getlength() * 2 - num_nonfrees * 3
+ @given(strategies.lists(strategies.integers(min_value=1, max_value=5)))
+ def test_direct_move_to_end(self, lst):
+ DICT = self._get_int_dict()
+ ll_d = rordereddict.ll_newdict(DICT)
+ rordereddict.ll_dict_setitem(ll_d, 1, 11)
+ rordereddict.ll_dict_setitem(ll_d, 2, 22)
+ def content():
+ return [(entry.key, entry.value) for entry in self._ll_iter(ll_d)]
+ for case in lst:
+ if case == 1:
+ rordereddict.ll_dict_move_to_end(ll_d, 1, True)
+ assert content() == [(2, 22), (1, 11)]
+ elif case == 2:
+ rordereddict.ll_dict_move_to_end(ll_d, 2, True)
+ assert content() == [(1, 11), (2, 22)]
+ elif case == 3:
+ py.test.raises(KeyError, rordereddict.ll_dict_move_to_end,
+ ll_d, 3, True)
+ elif case == 4:
+ rordereddict.ll_dict_move_to_end(ll_d, 2, False)
+ assert content() == [(2, 22), (1, 11)]
+ elif case == 5:
+ rordereddict.ll_dict_move_to_end(ll_d, 1, False)
+ assert content() == [(1, 11), (2, 22)]
+
class TestRDictDirectDummyKey(TestRDictDirect):
class dummykeyobj:
@@ -369,10 +399,29 @@
res = self.interpret(func, [5])
assert res == 6
+ def test_move_to_end(self):
+ def func():
+ d1 = OrderedDict()
+ d1['key1'] = 'value1'
+ d1['key2'] = 'value2'
+ for i in range(20):
+ objectmodel.move_to_end(d1, 'key1')
+ assert d1.keys() == ['key2', 'key1']
+ objectmodel.move_to_end(d1, 'key2')
+ assert d1.keys() == ['key1', 'key2']
+ for i in range(20):
+ objectmodel.move_to_end(d1, 'key2', last=False)
+ assert d1.keys() == ['key2', 'key1']
+ objectmodel.move_to_end(d1, 'key1', last=False)
+ assert d1.keys() == ['key1', 'key2']
+ func()
+ self.interpret(func, [])
+
class ODictSpace(MappingSpace):
MappingRepr = rodct.OrderedDictRepr
new_reference = OrderedDict
+ moved_around = False
ll_getitem = staticmethod(rodct.ll_dict_getitem)
ll_setitem = staticmethod(rodct.ll_dict_setitem)
ll_delitem = staticmethod(rodct.ll_dict_delitem)
@@ -417,9 +466,23 @@
def removeindex(self):
# remove the index, as done during translation for prebuilt dicts
# (but cannot be done if we already removed a key)
- if not self.removed_keys:
+ if not self.removed_keys and not self.moved_around:
rodct.ll_no_initial_index(self.l_dict)
+ def move_to_end(self, key, last=True):
+ ll_key = self.ll_key(key)
+ rodct.ll_dict_move_to_end(self.l_dict, ll_key, last)
+ value = self.reference.pop(key)
+ if last:
+ self.reference[key] = value
+ else:
+ items = self.reference.items()
+ self.reference.clear()
+ self.reference[key] = value
+ self.reference.update(items)
+ # prevent ll_no_initial_index()
+ self.moved_around = True
+
def fullcheck(self):
# overridden to also check key order
assert self.ll_len(self.l_dict) == len(self.reference)
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
