Author: Manuel Jacob <m...@manueljacob.de> Branch: py3k Changeset: r82462:842942cabb34 Date: 2016-02-23 22:26 +0100 http://bitbucket.org/pypy/pypy/changeset/842942cabb34/
Log: hg merge default diff too long, truncating to 2000 out of 5375 lines diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -7,6 +7,7 @@ content = fid.read() # from cffi's Verifier() key = '\x00'.join([sys.version[:3], content]) + key += 'cpyext-gc-support-2' # this branch requires recompilation! if sys.version_info >= (3,): key = key.encode('utf-8') k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) diff --git a/pypy/doc/discussion/rawrefcount.rst b/pypy/doc/discussion/rawrefcount.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/discussion/rawrefcount.rst @@ -0,0 +1,158 @@ +====================== +Rawrefcount and the GC +====================== + + +GC Interface +------------ + +"PyObject" is a raw structure with at least two fields, ob_refcnt and +ob_pypy_link. The ob_refcnt is the reference counter as used on +CPython. If the PyObject structure is linked to a live PyPy object, +its current address is stored in ob_pypy_link and ob_refcnt is bumped +by either the constant REFCNT_FROM_PYPY, or the constant +REFCNT_FROM_PYPY_LIGHT (== REFCNT_FROM_PYPY + SOME_HUGE_VALUE) +(to mean "light finalizer"). + +Most PyPy objects exist outside cpyext, and conversely in cpyext it is +possible that a lot of PyObjects exist without being seen by the rest +of PyPy. At the interface, however, we can "link" a PyPy object and a +PyObject. There are two kinds of link: + +rawrefcount.create_link_pypy(p, ob) + + Makes a link between an exising object gcref 'p' and a newly + allocated PyObject structure 'ob'. ob->ob_refcnt must be + initialized to either REFCNT_FROM_PYPY, or + REFCNT_FROM_PYPY_LIGHT. (The second case is an optimization: + when the GC finds the PyPy object and PyObject no longer + referenced, it can just free() the PyObject.) + +rawrefcount.create_link_pyobj(p, ob) + + Makes a link from an existing PyObject structure 'ob' to a newly + allocated W_CPyExtPlaceHolderObject 'p'. You must also add + REFCNT_FROM_PYPY to ob->ob_refcnt. For cases where the PyObject + contains all the data, and the PyPy object is just a proxy. The + W_CPyExtPlaceHolderObject should have only a field that contains + the address of the PyObject, but that's outside the scope of the + GC. + +rawrefcount.from_obj(p) + + If there is a link from object 'p' made with create_link_pypy(), + returns the corresponding 'ob'. Otherwise, returns NULL. + +rawrefcount.to_obj(Class, ob) + + Returns ob->ob_pypy_link, cast to an instance of 'Class'. + + +Collection logic +---------------- + +Objects existing purely on the C side have ob->ob_pypy_link == 0; +these are purely reference counted. On the other hand, if +ob->ob_pypy_link != 0, then ob->ob_refcnt is at least REFCNT_FROM_PYPY +and the object is part of a "link". + +The idea is that links whose 'p' is not reachable from other PyPy +objects *and* whose 'ob->ob_refcnt' is REFCNT_FROM_PYPY or +REFCNT_FROM_PYPY_LIGHT are the ones who die. But it is more messy +because PyObjects still (usually) need to have a tp_dealloc called, +and this cannot occur immediately (and can do random things like +accessing other references this object points to, or resurrecting the +object). + +Let P = list of links created with rawrefcount.create_link_pypy() +and O = list of links created with rawrefcount.create_link_pyobj(). +The PyPy objects in the list O are all W_CPyExtPlaceHolderObject: all +the data is in the PyObjects, and all outsite references (if any) are +in C, as "PyObject *" fields. + +So, during the collection we do this about P links: + + for (p, ob) in P: + if ob->ob_refcnt != REFCNT_FROM_PYPY + and ob->ob_refcnt != REFCNT_FROM_PYPY_LIGHT: + mark 'p' as surviving, as well as all its dependencies + +At the end of the collection, the P and O links are both handled like +this: + + for (p, ob) in P + O: + if p is not surviving: # even if 'ob' might be surviving + unlink p and ob + if ob->ob_refcnt == REFCNT_FROM_PYPY_LIGHT: + free(ob) + elif ob->ob_refcnt > REFCNT_FROM_PYPY_LIGHT: + ob->ob_refcnt -= REFCNT_FROM_PYPY_LIGHT + else: + ob->ob_refcnt -= REFCNT_FROM_PYPY + if ob->ob_refcnt == 0: + invoke _Py_Dealloc(ob) later, outside the GC + + +GC Implementation +----------------- + +We need two copies of both the P list and O list, for young or old +objects. All four lists can be regular AddressLists of 'ob' objects. + +We also need an AddressDict mapping 'p' to 'ob' for all links in the P +list, and update it when PyPy objects move. + + +Further notes +------------- + +XXX +XXX the rest is the ideal world, but as a first step, we'll look +XXX for the minimal tweaks needed to adapt the existing cpyext +XXX + +For objects that are opaque in CPython, like <dict>, we always create +a PyPy object, and then when needed we make an empty PyObject and +attach it with create_link_pypy()/REFCNT_FROM_PYPY_LIGHT. + +For <int> and <float> objects, the corresponding PyObjects contain a +"long" or "double" field too. We link them with create_link_pypy() +and we can use REFCNT_FROM_PYPY_LIGHT too: 'tp_dealloc' doesn't +need to be called, and instead just calling free() is fine. + +For <type> objects, we need both a PyPy and a PyObject side. These +are made with create_link_pypy()/REFCNT_FROM_PYPY. + +For custom PyXxxObjects allocated from the C extension module, we +need create_link_pyobj(). + +For <str> or <unicode> objects coming from PyPy, we use +create_link_pypy()/REFCNT_FROM_PYPY_LIGHT with a PyObject +preallocated with the size of the string. We copy the string +lazily into that area if PyString_AS_STRING() is called. + +For <str>, <unicode>, <tuple> or <list> objects in the C extension +module, we first allocate it as only a PyObject, which supports +mutation of the data from C, like CPython. When it is exported to +PyPy we could make a W_CPyExtPlaceHolderObject with +create_link_pyobj(). + +For <tuple> objects coming from PyPy, if they are not specialized, +then the PyPy side holds a regular reference to the items. Then we +can allocate a PyTupleObject and store in it borrowed PyObject +pointers to the items. Such a case is created with +create_link_pypy()/REFCNT_FROM_PYPY_LIGHT. If it is specialized, +then it doesn't work because the items are created just-in-time on the +PyPy side. In this case, the PyTupleObject needs to hold real +references to the PyObject items, and we use create_link_pypy()/ +REFCNT_FROM_PYPY. In all cases, we have a C array of PyObjects +that we can directly return from PySequence_Fast_ITEMS, PyTuple_ITEMS, +PyTuple_GetItem, and so on. + +For <list> objects coming from PyPy, we can use a cpyext list +strategy. The list turns into a PyListObject, as if it had been +allocated from C in the first place. The special strategy can hold +(only) a direct reference to the PyListObject, and we can use either +create_link_pyobj() or create_link_pypy() (to be decided). +PySequence_Fast_ITEMS then works for lists too, and PyList_GetItem +can return a borrowed reference, and so on. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -51,7 +51,7 @@ class W_Root(object): """This is the abstract root class of all wrapped objects that live in a 'normal' object space like StdObjSpace.""" - __slots__ = () + __slots__ = ('__weakref__',) user_overridden_class = False def getdict(self, space): diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -248,7 +248,7 @@ def user_setup(self, space, w_subtype): self.space = space self.w__class__ = w_subtype - self.user_setup_slots(w_subtype.nslots) + self.user_setup_slots(w_subtype.layout.nslots) def user_setup_slots(self, nslots): assert nslots == 0 diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -58,7 +58,6 @@ import pypy.module.cpyext.funcobject import pypy.module.cpyext.frameobject import pypy.module.cpyext.classobject -import pypy.module.cpyext.pypyintf import pypy.module.cpyext.exception import pypy.module.cpyext.memoryobject import pypy.module.cpyext.codecs diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -9,7 +9,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import ll2ctypes from rpython.rtyper.annlowlevel import llhelper -from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here from rpython.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.gensupp import NameManager @@ -29,13 +29,13 @@ from rpython.rlib.rposix import is_valid_fd, validate_fd from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize -from rpython.rlib.exports import export_struct from pypy.module import exceptions from pypy.module.exceptions import interp_exceptions # CPython 2.4 compatibility from py.builtin import BaseException from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rlib import rawrefcount DEBUG_WRAPPER = True @@ -194,7 +194,7 @@ class ApiFunction: def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED, - c_name=None, gil=None): + c_name=None, gil=None, result_borrowed=False): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) @@ -211,6 +211,11 @@ self.argnames = argnames[1:] assert len(self.argnames) == len(self.argtypes) self.gil = gil + self.result_borrowed = result_borrowed + # + def get_llhelper(space): + return llhelper(self.functype, self.get_wrapper(space)) + self.get_llhelper = get_llhelper def __repr__(self): return "<cpyext function %s>" % (self.callable.__name__,) @@ -218,13 +223,6 @@ def _freeze_(self): return True - def get_llhelper(self, space): - llh = getattr(self, '_llhelper', None) - if llh is None: - llh = llhelper(self.functype, self.get_wrapper(space)) - self._llhelper = llh - return llh - @specialize.memo() def get_wrapper(self, space): wrapper = getattr(self, '_wrapper', None) @@ -237,7 +235,7 @@ return wrapper def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', - gil=None): + gil=None, result_borrowed=False): """ Declares a function to be exported. - `argtypes`, `restype` are lltypes and describe the function signature. @@ -266,13 +264,15 @@ rffi.cast(restype, 0) == 0) def decorate(func): + func._always_inline_ = 'try' func_name = func.func_name if header is not None: c_name = None else: c_name = func_name api_function = ApiFunction(argtypes, restype, func, error, - c_name=c_name, gil=gil) + c_name=c_name, gil=gil, + result_borrowed=result_borrowed) func.api_func = api_function if header is not None: @@ -283,6 +283,10 @@ raise ValueError("function %s has no return value for exceptions" % func) def make_unwrapper(catch_exception): + # ZZZ is this whole logic really needed??? It seems to be only + # for RPython code calling PyXxx() functions directly. I would + # think that usually directly calling the function is clean + # enough now names = api_function.argnames types_names_enum_ui = unrolling_iterable(enumerate( zip(api_function.argtypes, @@ -290,56 +294,58 @@ @specialize.ll() def unwrapper(space, *args): - from pypy.module.cpyext.pyobject import Py_DecRef - from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import Reference + from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj + from pypy.module.cpyext.pyobject import from_ref, as_pyobj newargs = () - to_decref = [] + keepalives = () assert len(args) == len(api_function.argtypes) for i, (ARG, is_wrapped) in types_names_enum_ui: input_arg = args[i] if is_PyObject(ARG) and not is_wrapped: - # build a reference - if input_arg is None: - arg = lltype.nullptr(PyObject.TO) - elif isinstance(input_arg, W_Root): - ref = make_ref(space, input_arg) - to_decref.append(ref) - arg = rffi.cast(ARG, ref) + # build a 'PyObject *' (not holding a reference) + if not is_pyobj(input_arg): + keepalives += (input_arg,) + arg = rffi.cast(ARG, as_pyobj(space, input_arg)) + else: + arg = rffi.cast(ARG, input_arg) + elif is_PyObject(ARG) and is_wrapped: + # build a W_Root, possibly from a 'PyObject *' + if is_pyobj(input_arg): + arg = from_ref(space, input_arg) else: arg = input_arg - elif is_PyObject(ARG) and is_wrapped: - # convert to a wrapped object - if input_arg is None: - arg = input_arg - elif isinstance(input_arg, W_Root): - arg = input_arg - else: - try: - arg = from_ref(space, - rffi.cast(PyObject, input_arg)) - except TypeError, e: - err = OperationError(space.w_TypeError, - space.wrap( - "could not cast arg to PyObject")) - if not catch_exception: - raise err - state = space.fromcache(State) - state.set_exception(err) - if is_PyObject(restype): - return None - else: - return api_function.error_value + + ## ZZZ: for is_pyobj: + ## try: + ## arg = from_ref(space, + ## rffi.cast(PyObject, input_arg)) + ## except TypeError, e: + ## err = OperationError(space.w_TypeError, + ## space.wrap( + ## "could not cast arg to PyObject")) + ## if not catch_exception: + ## raise err + ## state = space.fromcache(State) + ## state.set_exception(err) + ## if is_PyObject(restype): + ## return None + ## else: + ## return api_function.error_value else: - # convert to a wrapped object + # arg is not declared as PyObject, no magic arg = input_arg newargs += (arg, ) - try: + if not catch_exception: + try: + res = func(space, *newargs) + finally: + keepalive_until_here(*keepalives) + else: + # non-rpython variant + assert not we_are_translated() try: res = func(space, *newargs) except OperationError, e: - if not catch_exception: - raise if not hasattr(api_function, "error_value"): raise state = space.fromcache(State) @@ -348,21 +354,13 @@ return None else: return api_function.error_value - if not we_are_translated(): - got_integer = isinstance(res, (int, long, float)) - assert got_integer == expect_integer,'got %r not integer' % res - if res is None: - return None - elif isinstance(res, Reference): - return res.get_wrapped(space) - else: - return res - finally: - for arg in to_decref: - Py_DecRef(space, arg) + # 'keepalives' is alive here (it's not rpython) + got_integer = isinstance(res, (int, long, float)) + assert got_integer == expect_integer, ( + 'got %r not integer' % (res,)) + return res unwrapper.func = func unwrapper.api_func = api_function - unwrapper._always_inline_ = 'try' return unwrapper unwrapper_catch = make_unwrapper(True) @@ -501,7 +499,7 @@ GLOBALS['%s#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) for cpyname in '''PyMethodObject PyListObject PyLongObject - PyDictObject PyTupleObject'''.split(): + PyDictObject'''.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s' % (cpyname, )) build_exported_objects() @@ -512,14 +510,16 @@ return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr, "PyDateTime_CAPI*": lltype.Ptr(PyDateTime_CAPI)}[ctype] +# Note: as a special case, "PyObject" is the pointer type in RPython, +# corresponding to "PyObject *" in C. We do that only for PyObject. +# For example, "PyTypeObject" is the struct type even in RPython. PyTypeObject = lltype.ForwardReference() PyTypeObjectPtr = lltype.Ptr(PyTypeObject) -# It is important that these PyObjects are allocated in a raw fashion -# Thus we cannot save a forward pointer to the wrapped object -# So we need a forward and backward mapping in our State instance PyObjectStruct = lltype.ForwardReference() PyObject = lltype.Ptr(PyObjectStruct) -PyObjectFields = (("ob_refcnt", lltype.Signed), ("ob_type", PyTypeObjectPtr)) +PyObjectFields = (("ob_refcnt", lltype.Signed), + ("ob_pypy_link", lltype.Signed), + ("ob_type", PyTypeObjectPtr)) PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), ) cpython_struct('PyObject', PyObjectFields, PyObjectStruct) PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) @@ -616,8 +616,8 @@ @specialize.ll() def wrapper(*args): - from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import Reference + from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj + from pypy.module.cpyext.pyobject import as_pyobj # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer if gil_acquire: @@ -626,6 +626,7 @@ llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py retval = fatal_value boxed_args = () + tb = None try: if not we_are_translated() and DEBUG_WRAPPER: print >>sys.stderr, callable, @@ -633,10 +634,8 @@ for i, (typ, is_wrapped) in argtypes_enum_ui: arg = args[i] if is_PyObject(typ) and is_wrapped: - if arg: - arg_conv = from_ref(space, rffi.cast(PyObject, arg)) - else: - arg_conv = None + assert is_pyobj(arg) + arg_conv = from_ref(space, rffi.cast(PyObject, arg)) else: arg_conv = arg boxed_args += (arg_conv, ) @@ -651,6 +650,7 @@ except BaseException, e: failed = True if not we_are_translated(): + tb = sys.exc_info()[2] message = repr(e) import traceback traceback.print_exc() @@ -669,29 +669,34 @@ retval = error_value elif is_PyObject(callable.api_func.restype): - if result is None: - retval = rffi.cast(callable.api_func.restype, - make_ref(space, None)) - elif isinstance(result, Reference): - retval = result.get_ref(space) - elif not rffi._isllptr(result): - retval = rffi.cast(callable.api_func.restype, - make_ref(space, result)) + if is_pyobj(result): + retval = result else: - retval = result + if result is not None: + if callable.api_func.result_borrowed: + retval = as_pyobj(space, result) + else: + retval = make_ref(space, result) + retval = rffi.cast(callable.api_func.restype, retval) + else: + retval = lltype.nullptr(PyObject.TO) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) except Exception, e: print 'Fatal error in cpyext, CPython compatibility layer, calling', callable.__name__ print 'Either report a bug or consider not using this particular extension' if not we_are_translated(): + if tb is None: + tb = sys.exc_info()[2] import traceback traceback.print_exc() - print str(e) + if sys.stdout == sys.__stdout__: + import pdb; pdb.post_mortem(tb) # we can't do much here, since we're in ctypes, swallow else: print str(e) pypy_debug_catch_fatal_exception() + assert False rffi.stackcounter.stacks_counter -= 1 if gil_release: rgil.release() @@ -825,6 +830,19 @@ outputfilename=str(udir / "module_cache" / "pypyapi")) modulename = py.path.local(eci.libraries[-1]) + def dealloc_trigger(): + from pypy.module.cpyext.pyobject import _Py_Dealloc + print 'dealloc_trigger...' + while True: + ob = rawrefcount.next_dead(PyObject) + if not ob: + break + print ob + _Py_Dealloc(space, ob) + print 'dealloc_trigger DONE' + return "RETRY" + rawrefcount.init(dealloc_trigger) + run_bootstrap_functions(space) # load the bridge, and init structure @@ -834,9 +852,9 @@ space.fromcache(State).install_dll(eci) # populate static data - builder = StaticObjectBuilder(space) + builder = space.fromcache(StaticObjectBuilder) for name, (typ, expr) in GLOBALS.iteritems(): - from pypy.module import cpyext + from pypy.module import cpyext # for the eval() below w_obj = eval(expr) if name.endswith('#'): name = name[:-1] @@ -892,27 +910,44 @@ class StaticObjectBuilder: def __init__(self, space): self.space = space - self.to_attach = [] + self.static_pyobjs = [] + self.static_objs_w = [] + self.cpyext_type_init = None + # + # add a "method" that is overridden in setup_library() + # ('self.static_pyobjs' is completely ignored in that case) + self.get_static_pyobjs = lambda: self.static_pyobjs def prepare(self, py_obj, w_obj): - from pypy.module.cpyext.pyobject import track_reference - py_obj.c_ob_refcnt = 1 - track_reference(self.space, py_obj, w_obj) - self.to_attach.append((py_obj, w_obj)) + "NOT_RPYTHON" + if py_obj: + py_obj.c_ob_refcnt = 1 # 1 for kept immortal + self.static_pyobjs.append(py_obj) + self.static_objs_w.append(w_obj) def attach_all(self): + # this is RPython, called once in pypy-c when it imports cpyext from pypy.module.cpyext.pyobject import get_typedescr, make_ref from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2 + from pypy.module.cpyext.pyobject import track_reference + # space = self.space - space._cpyext_type_init = [] - for py_obj, w_obj in self.to_attach: + static_pyobjs = self.get_static_pyobjs() + static_objs_w = self.static_objs_w + for i in range(len(static_objs_w)): + track_reference(space, static_pyobjs[i], static_objs_w[i]) + # + self.cpyext_type_init = [] + for i in range(len(static_objs_w)): + py_obj = static_pyobjs[i] + w_obj = static_objs_w[i] w_type = space.type(w_obj) - typedescr = get_typedescr(w_type.instancetypedef) + typedescr = get_typedescr(w_type.layout.typedef) py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) typedescr.attach(space, py_obj, w_obj) - cpyext_type_init = space._cpyext_type_init - del space._cpyext_type_init + cpyext_type_init = self.cpyext_type_init + self.cpyext_type_init = None for pto, w_type in cpyext_type_init: finish_type_1(space, pto) finish_type_2(space, pto, w_type) @@ -1064,7 +1099,7 @@ if name.endswith('#'): structs.append('%s %s;' % (typ[:-1], name[:-1])) elif name.startswith('PyExc_'): - structs.append('extern PyTypeObject _%s;' % (name,)) + structs.append('PyTypeObject _%s;' % (name,)) structs.append('PyObject* %s = (PyObject*)&_%s;' % (name, name)) elif typ == 'PyDateTime_CAPI*': structs.append('%s %s = NULL;' % (typ, name)) @@ -1116,10 +1151,8 @@ def setup_library(space): "NOT_RPYTHON" - from pypy.module.cpyext.pyobject import make_ref use_micronumpy = setup_micronumpy(space) - - export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) + export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS) from rpython.translator.c.database import LowLevelDatabase db = LowLevelDatabase() @@ -1135,41 +1168,37 @@ run_bootstrap_functions(space) setup_va_functions(eci) - from pypy.module import cpyext # for eval() below - - # Set up the types. Needs a special case, because of the - # immediate cycle involving 'c_ob_type', and because we don't - # want these types to be Py_TPFLAGS_HEAPTYPE. - static_types = {} - for name, (typ, expr) in GLOBALS.items(): - if typ == 'PyTypeObject*': - pto = lltype.malloc(PyTypeObject, immortal=True, - zero=True, flavor='raw') - pto.c_ob_refcnt = 1 - pto.c_tp_basicsize = -1 - static_types[name] = pto - builder = StaticObjectBuilder(space) - for name, pto in static_types.items(): - pto.c_ob_type = static_types['PyType_Type#'] - w_type = eval(GLOBALS[name][1]) - builder.prepare(rffi.cast(PyObject, pto), w_type) - builder.attach_all() - - # populate static data - for name, (typ, expr) in GLOBALS.iteritems(): - name = name.replace("#", "") - if name.startswith('PyExc_'): + # emit uninitialized static data + builder = space.fromcache(StaticObjectBuilder) + lines = ['PyObject *pypy_static_pyobjs[] = {\n'] + include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n'] + for name, (typ, expr) in sorted(GLOBALS.items()): + if name.endswith('#'): + assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*') + typ, name = typ[:-1], name[:-1] + elif name.startswith('PyExc_'): + typ = 'PyTypeObject' name = '_' + name - w_obj = eval(expr) - if typ in ('PyObject*', 'PyTypeObject*'): - struct_ptr = make_ref(space, w_obj) elif typ == 'PyDateTime_CAPI*': continue else: assert False, "Unknown static data: %s %s" % (typ, name) - struct = rffi.cast(get_structtype_for_ctype(typ), struct_ptr)._obj - struct._compilation_info = eci - export_struct(name, struct) + + from pypy.module import cpyext # for the eval() below + w_obj = eval(expr) + builder.prepare(None, w_obj) + lines.append('\t(PyObject *)&%s,\n' % (name,)) + include_lines.append('RPY_EXPORTED %s %s;\n' % (typ, name)) + + lines.append('};\n') + eci2 = CConfig._compilation_info_.merge(ExternalCompilationInfo( + separate_module_sources = [''.join(lines)], + post_include_bits = [''.join(include_lines)], + )) + # override this method to return a pointer to this C array directly + builder.get_static_pyobjs = rffi.CExternVariable( + PyObjectP, 'pypy_static_pyobjs', eci2, c_type='PyObject **', + getter_only=True, declare_as_extern=False) for name, func in FUNCTIONS.iteritems(): newname = mangle_name('PyPy', name) or name @@ -1180,6 +1209,10 @@ trunk_include = pypydir.dirpath() / 'include' copy_header_files(trunk_include, use_micronumpy) +def init_static_data_translated(space): + builder = space.fromcache(StaticObjectBuilder) + builder.attach_all() + def _load_from_cffi(space, name, path, initptr): from pypy.module._cffi_backend import cffi1_module cffi1_module.load_cffi1_module(space, name, path, initptr) @@ -1262,22 +1295,18 @@ @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, True, False)(space, func, *args) - -@specialize.ll() -def generic_cpy_call_dont_decref(space, func, *args): - FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, False, False)(space, func, *args) + return make_generic_cpy_call(FT, False)(space, func, *args) @specialize.ll() def generic_cpy_call_expect_null(space, func, *args): FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, True, True)(space, func, *args) + return make_generic_cpy_call(FT, True)(space, func, *args) @specialize.memo() -def make_generic_cpy_call(FT, decref_args, expect_null): +def make_generic_cpy_call(FT, expect_null): from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef - from pypy.module.cpyext.pyobject import RefcountState + from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj + from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.module.cpyext.pyerrors import PyErr_Occurred unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS)) RESULT_TYPE = FT.RESULT @@ -1305,65 +1334,49 @@ @specialize.ll() def generic_cpy_call(space, func, *args): boxed_args = () - to_decref = [] + keepalives = () assert len(args) == len(FT.ARGS) for i, ARG in unrolling_arg_types: arg = args[i] if is_PyObject(ARG): - if arg is None: - boxed_args += (lltype.nullptr(PyObject.TO),) - elif isinstance(arg, W_Root): - ref = make_ref(space, arg) - boxed_args += (ref,) - if decref_args: - to_decref.append(ref) - else: - boxed_args += (arg,) - else: - boxed_args += (arg,) + if not is_pyobj(arg): + keepalives += (arg,) + arg = as_pyobj(space, arg) + boxed_args += (arg,) try: - # create a new container for borrowed references - state = space.fromcache(RefcountState) - old_container = state.swap_borrow_container(None) - try: - # Call the function - result = call_external_function(func, *boxed_args) - finally: - state.swap_borrow_container(old_container) + # Call the function + result = call_external_function(func, *boxed_args) + finally: + keepalive_until_here(*keepalives) - if is_PyObject(RESULT_TYPE): - if result is None: - ret = result - elif isinstance(result, W_Root): - ret = result + if is_PyObject(RESULT_TYPE): + if not is_pyobj(result): + ret = result + else: + # The object reference returned from a C function + # that is called from Python must be an owned reference + # - ownership is transferred from the function to its caller. + if result: + ret = get_w_obj_and_decref(space, result) else: - ret = from_ref(space, result) - # The object reference returned from a C function - # that is called from Python must be an owned reference - # - ownership is transferred from the function to its caller. - if result: - Py_DecRef(space, result) + ret = None - # Check for exception consistency - has_error = PyErr_Occurred(space) is not None - has_result = ret is not None - if has_error and has_result: - raise OperationError(space.w_SystemError, space.wrap( - "An exception was set, but function returned a value")) - elif not expect_null and not has_error and not has_result: - raise OperationError(space.w_SystemError, space.wrap( - "Function returned a NULL result without setting an exception")) + # Check for exception consistency + has_error = PyErr_Occurred(space) is not None + has_result = ret is not None + if has_error and has_result: + raise OperationError(space.w_SystemError, space.wrap( + "An exception was set, but function returned a value")) + elif not expect_null and not has_error and not has_result: + raise OperationError(space.w_SystemError, space.wrap( + "Function returned a NULL result without setting an exception")) - if has_error: - state = space.fromcache(State) - state.check_and_raise_exception() + if has_error: + state = space.fromcache(State) + state.check_and_raise_exception() - return ret - return result - finally: - if decref_args: - for ref in to_decref: - Py_DecRef(space, ref) + return ret + return result + return generic_cpy_call - diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -59,7 +59,7 @@ @bootstrap_function def init_bytesobject(space): "Type description of PyBytesObject" - make_typedescr(space.w_str.instancetypedef, + make_typedescr(space.w_str.layout.typedef, basestruct=PyBytesObject.TO, attach=bytes_attach, dealloc=bytes_dealloc, @@ -69,11 +69,11 @@ def new_empty_str(space, length): """ - Allocates a PyBytesObject and its buffer, but without a corresponding + Allocate a PyBytesObject and its buffer, but without a corresponding interpreter object. The buffer may be mutated, until bytes_realize() is - called. + called. Refcount of the result is 1. """ - typedescr = get_typedescr(space.w_bytes.instancetypedef) + typedescr = get_typedescr(space.w_bytes.layout.typedef) py_obj = typedescr.allocate(space, space.w_bytes) py_str = rffi.cast(PyBytesObject, py_obj) diff --git a/pypy/module/cpyext/complexobject.py b/pypy/module/cpyext/complexobject.py --- a/pypy/module/cpyext/complexobject.py +++ b/pypy/module/cpyext/complexobject.py @@ -43,7 +43,7 @@ # lltype does not handle functions returning a structure. This implements a # helper function, which takes as argument a reference to the return value. -@cpython_api([PyObject, Py_complex_ptr], lltype.Void) +@cpython_api([PyObject, Py_complex_ptr], rffi.INT_real, error=-1) def _PyComplex_AsCComplex(space, w_obj, result): """Return the Py_complex value of the complex number op. @@ -60,7 +60,7 @@ # if the above did not work, interpret obj as a float giving the # real part of the result, and fill in the imaginary part as 0. result.c_real = PyFloat_AsDouble(space, w_obj) # -1 on failure - return + return 0 if not PyComplex_Check(space, w_obj): raise OperationError(space.w_TypeError, space.wrap( @@ -69,3 +69,4 @@ assert isinstance(w_obj, W_ComplexObject) result.c_real = w_obj.realval result.c_imag = w_obj.imagval + return 0 diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py --- a/pypy/module/cpyext/dictobject.py +++ b/pypy/module/cpyext/dictobject.py @@ -2,8 +2,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t, Py_ssize_tP, CONST_STRING) -from pypy.module.cpyext.pyobject import PyObject, PyObjectP, borrow_from -from pypy.module.cpyext.pyobject import RefcountState +from pypy.module.cpyext.pyobject import PyObject, PyObjectP, as_pyobj from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import specialize @@ -14,13 +13,17 @@ PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict") -@cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) +@cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL, + result_borrowed=True) def PyDict_GetItem(space, w_dict, w_key): try: w_res = space.getitem(w_dict, w_key) except: return None - return borrow_from(w_dict, w_res) + # NOTE: this works so far because all our dict strategies store + # *values* as full objects, which stay alive as long as the dict is + # alive and not modified. So we can return a borrowed ref. + return w_res @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_SetItem(space, w_dict, w_key, w_obj): @@ -47,7 +50,8 @@ else: PyErr_BadInternalCall(space) -@cpython_api([PyObject, CONST_STRING], PyObject, error=CANNOT_FAIL) +@cpython_api([PyObject, CONST_STRING], PyObject, error=CANNOT_FAIL, + result_borrowed=True) def PyDict_GetItemString(space, w_dict, key): """This is the same as PyDict_GetItem(), but key is specified as a char*, rather than a PyObject*.""" @@ -55,9 +59,10 @@ w_res = space.finditem_str(w_dict, rffi.charp2str(key)) except: w_res = None - if w_res is None: - return None - return borrow_from(w_dict, w_res) + # NOTE: this works so far because all our dict strategies store + # *values* as full objects, which stay alive as long as the dict is + # alive and not modified. So we can return a borrowed ref. + return w_res @cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) def PyDict_DelItemString(space, w_dict, key_ptr): @@ -170,10 +175,13 @@ if w_dict is None: return 0 - # Note: this is not efficient. Storing an iterator would probably + # XXX XXX PyDict_Next is not efficient. Storing an iterator would probably # work, but we can't work out how to not leak it if iteration does - # not complete. + # not complete. Alternatively, we could add some RPython-only + # dict-iterator method to move forward by N steps. + w_dict.ensure_object_strategy() # make sure both keys and values can + # be borrwed try: w_iter = space.iter(space.call_method(space.w_dict, "items", w_dict)) pos = ppos[0] @@ -183,11 +191,10 @@ w_item = space.next(w_iter) w_key, w_value = space.fixedview(w_item, 2) - state = space.fromcache(RefcountState) if pkey: - pkey[0] = state.make_borrowed(w_dict, w_key) + pkey[0] = as_pyobj(space, w_key) if pvalue: - pvalue[0] = state.make_borrowed(w_dict, w_value) + pvalue[0] = as_pyobj(space, w_value) ppos[0] += 1 except OperationError, e: if not e.match(space, space.w_StopIteration): diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -4,7 +4,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, cpython_struct, is_valid_fp) -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject from pypy.module.__builtin__ import compiling @@ -23,7 +23,7 @@ def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): return space.call(w_obj, w_arg, w_kwds) -@cpython_api([], PyObject) +@cpython_api([], PyObject, result_borrowed=True) def PyEval_GetBuiltins(space): """Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is @@ -36,25 +36,25 @@ w_builtins = w_builtins.getdict(space) else: w_builtins = space.builtin.getdict(space) - return borrow_from(None, w_builtins) + return w_builtins # borrowed ref in all cases -@cpython_api([], PyObject, error=CANNOT_FAIL) +@cpython_api([], PyObject, error=CANNOT_FAIL, result_borrowed=True) def PyEval_GetLocals(space): """Return a dictionary of the local variables in the current execution frame, or NULL if no frame is currently executing.""" caller = space.getexecutioncontext().gettopframe_nohidden() if caller is None: return None - return borrow_from(None, caller.getdictscope()) + return caller.getdictscope() # borrowed ref -@cpython_api([], PyObject, error=CANNOT_FAIL) +@cpython_api([], PyObject, error=CANNOT_FAIL, result_borrowed=True) def PyEval_GetGlobals(space): """Return a dictionary of the global variables in the current execution frame, or NULL if no frame is currently executing.""" caller = space.getexecutioncontext().gettopframe_nohidden() if caller is None: return None - return borrow_from(None, caller.get_w_globals()) + return caller.get_w_globals() # borrowed ref @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -3,7 +3,7 @@ PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t, cpython_api, bootstrap_function, cpython_struct, build_type_checkers) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, borrow_from) + PyObject, make_ref, from_ref, Py_DecRef, make_typedescr) from rpython.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError from pypy.interpreter.function import Function, Method @@ -83,12 +83,12 @@ from pypy.module.cpyext.object import PyObject_dealloc PyObject_dealloc(space, py_obj) -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): """Return the code object associated with the function object op.""" func = space.interp_w(Function, w_func) w_code = space.wrap(func.code) - return borrow_from(w_func, w_code) + return w_code # borrowed ref @cpython_api([PyObject, PyObject], PyObject) def PyMethod_New(space, w_func, w_self): @@ -98,18 +98,18 @@ not be NULL.""" return Method(space, w_func, w_self) -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyMethod_Function(space, w_method): """Return the function object associated with the method meth.""" assert isinstance(w_method, Method) - return borrow_from(w_method, w_method.w_function) + return w_method.w_function # borrowed ref -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyMethod_Self(space, w_method): """Return the instance associated with the method meth if it is bound, otherwise return NULL.""" assert isinstance(w_method, Method) - return borrow_from(w_method, w_method.w_instance) + return w_method.w_instance # borrowed ref def unwrap_list_of_strings(space, w_list): return [space.str_w(w_item) for w_item in space.fixedview(w_list)] diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py --- a/pypy/module/cpyext/import_.py +++ b/pypy/module/cpyext/import_.py @@ -1,7 +1,6 @@ from pypy.interpreter import module from pypy.module.cpyext.api import ( generic_cpy_call, cpython_api, PyObject, CONST_STRING) -from pypy.module.cpyext.pyobject import borrow_from from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError from pypy.interpreter.module import Module @@ -56,7 +55,7 @@ from pypy.module.imp.importing import reload return reload(space, w_mod) -@cpython_api([CONST_STRING], PyObject) +@cpython_api([CONST_STRING], PyObject, result_borrowed=True) def PyImport_AddModule(space, name): """Return the module object corresponding to a module name. The name argument may be of the form package.module. First check the modules @@ -71,19 +70,19 @@ not already present.""" from pypy.module.imp.importing import check_sys_modules_w modulename = rffi.charp2str(name) - w_modulename = space.wrap(modulename) w_mod = check_sys_modules_w(space, modulename) if not w_mod or space.is_w(w_mod, space.w_None): - w_mod = Module(space, w_modulename) - space.setitem(space.sys.get('modules'), w_modulename, w_mod) - return borrow_from(None, w_mod) + w_mod = Module(space, space.wrap(modulename)) + space.setitem(space.sys.get('modules'), space.wrap(modulename), w_mod) + # return a borrowed ref --- assumes one copy in sys.modules + return w_mod -@cpython_api([], PyObject) +@cpython_api([], PyObject, result_borrowed=True) def PyImport_GetModuleDict(space): """Return the dictionary used for the module administration (a.k.a. sys.modules). Note that this is a per-interpreter variable.""" w_modulesDict = space.sys.get('modules') - return borrow_from(None, w_modulesDict) + return w_modulesDict # borrowed ref @cpython_api([rffi.CCHARP, PyObject], PyObject) def PyImport_ExecCodeModule(space, name, w_code): diff --git a/pypy/module/cpyext/include/complexobject.h b/pypy/module/cpyext/include/complexobject.h --- a/pypy/module/cpyext/include/complexobject.h +++ b/pypy/module/cpyext/include/complexobject.h @@ -15,7 +15,7 @@ } Py_complex; /* generated function */ -PyAPI_FUNC(void) _PyComplex_AsCComplex(PyObject *, Py_complex *); +PyAPI_FUNC(int) _PyComplex_AsCComplex(PyObject *, Py_complex *); PyAPI_FUNC(PyObject *) _PyComplex_FromCComplex(Py_complex *); Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj) diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -17,7 +17,8 @@ #define staticforward static #define PyObject_HEAD \ - long ob_refcnt; \ + Py_ssize_t ob_refcnt; \ + Py_ssize_t ob_pypy_link; \ struct _typeobject *ob_type; #define PyObject_VAR_HEAD \ @@ -25,7 +26,7 @@ Py_ssize_t ob_size; /* Number of items in variable part */ #define PyObject_HEAD_INIT(type) \ - 1, type, + 1, 0, type, #define PyVarObject_HEAD_INIT(type, size) \ PyObject_HEAD_INIT(type) size, @@ -40,19 +41,19 @@ #ifdef PYPY_DEBUG_REFCOUNT /* Slow version, but useful for debugging */ -#define Py_INCREF(ob) (Py_IncRef((PyObject *)ob)) -#define Py_DECREF(ob) (Py_DecRef((PyObject *)ob)) -#define Py_XINCREF(ob) (Py_IncRef((PyObject *)ob)) -#define Py_XDECREF(ob) (Py_DecRef((PyObject *)ob)) +#define Py_INCREF(ob) (Py_IncRef((PyObject *)(ob))) +#define Py_DECREF(ob) (Py_DecRef((PyObject *)(ob))) +#define Py_XINCREF(ob) (Py_IncRef((PyObject *)(ob))) +#define Py_XDECREF(ob) (Py_DecRef((PyObject *)(ob))) #else /* Fast version */ -#define Py_INCREF(ob) (((PyObject *)ob)->ob_refcnt++) -#define Py_DECREF(ob) \ +#define Py_INCREF(ob) (((PyObject *)(ob))->ob_refcnt++) +#define Py_DECREF(op) \ do { \ - if (((PyObject *)ob)->ob_refcnt > 1) \ - ((PyObject *)ob)->ob_refcnt--; \ + if (--((PyObject *)(op))->ob_refcnt != 0) \ + ; \ else \ - Py_DecRef((PyObject *)ob); \ + _Py_Dealloc((PyObject *)(op)); \ } while (0) #define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -30,6 +30,13 @@ /* PyPy version as a string */ #define PYPY_VERSION "4.1.0-alpha0" +#define PYPY_VERSION_NUM 0x04010000 + +/* Defined to mean a PyPy where cpyext holds more regular references + to PyObjects, e.g. staying alive as long as the internal PyPy object + stays alive. */ +#define PYPY_CPYEXT_GC 1 +#define PyPy_Borrow(a, b) ((void) 0) /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -7,11 +7,21 @@ extern "C" { #endif +typedef struct { + PyObject_HEAD + Py_ssize_t ob_size; + PyObject **ob_item; /* XXX optimize to ob_item[] */ +} PyTupleObject; + /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); -#define PyTuple_SET_ITEM PyTuple_SetItem -#define PyTuple_GET_ITEM PyTuple_GetItem +/* Macro, trading safety for speed */ +#define PyTuple_GET_ITEM(op, i) (((PyTupleObject *)(op))->ob_item[i]) +#define PyTuple_GET_SIZE(op) Py_SIZE(op) + +/* Macro, *only* to be used to fill in brand new tuples */ +#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v) #ifdef __cplusplus diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py --- a/pypy/module/cpyext/listobject.py +++ b/pypy/module/cpyext/listobject.py @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t, build_type_checkers) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall -from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, borrow_from +from pypy.module.cpyext.pyobject import Py_DecRef, PyObject from pypy.objspace.std.listobject import W_ListObject from pypy.interpreter.error import OperationError @@ -38,7 +38,7 @@ w_list.setitem(index, w_item) return 0 -@cpython_api([PyObject, Py_ssize_t], PyObject) +@cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True) def PyList_GetItem(space, w_list, index): """Return the object at position pos in the list pointed to by p. The position must be positive, indexing from the end of the list is not @@ -49,8 +49,10 @@ if index < 0 or index >= w_list.length(): raise OperationError(space.w_IndexError, space.wrap( "list index out of range")) - w_item = w_list.getitem(index) - return borrow_from(w_list, w_item) + w_list.ensure_object_strategy() # make sure we can return a borrowed obj + # XXX ^^^ how does this interact with CPyListStrategy? + w_res = w_list.getitem(index) + return w_res # borrowed ref @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, @@ -101,12 +101,12 @@ return int(space.is_w(w_type, w_obj_type) or space.is_true(space.issubtype(w_obj_type, w_type))) -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyModule_GetDict(space, w_mod): if PyModule_Check(space, w_mod): assert isinstance(w_mod, Module) w_dict = w_mod.getdict(space) - return borrow_from(w_mod, w_dict) + return w_dict # borrowed reference, likely from w_mod.w_dict else: PyErr_BadInternalCall(space) diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -6,7 +6,7 @@ Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, - track_reference, get_typedescr, _Py_NewReference, RefcountState) + get_typedescr, _Py_NewReference) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.typeobject import W_TypeObject @@ -35,9 +35,9 @@ def _PyObject_NewVar(space, type, itemcount): w_type = from_ref(space, rffi.cast(PyObject, type)) assert isinstance(w_type, W_TypeObject) - typedescr = get_typedescr(w_type.instancetypedef) + typedescr = get_typedescr(w_type.layout.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - py_obj.c_ob_refcnt = 0 + #py_obj.c_ob_refcnt = 0 --- will be set to 1 again by PyObject_Init{Var} if type.c_tp_itemsize == 0: w_obj = PyObject_Init(space, py_obj, type) else: diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -6,7 +6,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, from_ref, Py_DecRef, borrow_from) + PyObject, PyObjectP, make_ref, from_ref, Py_DecRef) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from rpython.rlib import rposix, jit @@ -28,12 +28,12 @@ """This is a shorthand for PyErr_SetObject(type, Py_None).""" PyErr_SetObject(space, w_type, space.w_None) -@cpython_api([], PyObject) +@cpython_api([], PyObject, result_borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) if state.operror is None: return None - return borrow_from(None, state.operror.w_type) + return state.operror.w_type # borrowed ref @cpython_api([], lltype.Void) def PyErr_Clear(space): diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, CONST_STRING, FILEP) -from pypy.module.cpyext.pyobject import PyObject, borrow_from + cpython_api, CONST_STRING, FILEP) +from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.object import Py_PRINT_RAW from pypy.interpreter.error import OperationError @@ -80,4 +80,5 @@ @cpython_api([PyObject], PyObject) def PyFile_Name(space, w_p): """Return the name of the file specified by p as a string object.""" - return borrow_from(w_p, space.getattr(w_p, space.wrap("name"))) + w_name = space.getattr(w_p, space.wrap("name")) + return w_name # borrowed ref, should be a W_StringObject from the file diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -2,15 +2,19 @@ from pypy.interpreter.baseobjspace import W_Root, SpaceCache from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.extregistry import ExtRegistryEntry from pypy.module.cpyext.api import ( cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR, - CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr) + CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject, + INTERPLEVEL_API) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.objectobject import W_ObjectObject from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rweakref import RWeakKeyDictionary +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper +from rpython.rlib import rawrefcount + #________________________________________________________ # type description @@ -28,13 +32,15 @@ def allocate(self, space, w_type, itemcount=0): # similar to PyType_GenericAlloc? # except that it's not related to any pypy object. + # this returns a PyObject with ob_refcnt == 1. - pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) + pytype = as_pyobj(space, w_type) + pytype = rffi.cast(PyTypeObjectPtr, pytype) + assert pytype # Don't increase refcount for non-heaptypes - if pytype: - flags = rffi.cast(lltype.Signed, pytype.c_tp_flags) - if not flags & Py_TPFLAGS_HEAPTYPE: - Py_DecRef(space, w_type) + flags = rffi.cast(lltype.Signed, pytype.c_tp_flags) + if flags & Py_TPFLAGS_HEAPTYPE: + Py_IncRef(space, w_type) if pytype: size = pytype.c_tp_basicsize @@ -42,6 +48,7 @@ size = rffi.sizeof(self.basestruct) if itemcount: size += itemcount * pytype.c_tp_itemsize + assert size >= rffi.sizeof(PyObject.TO) buf = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) pyobj = rffi.cast(PyObject, buf) @@ -56,9 +63,6 @@ w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) w_obj = space.allocate_instance(self.W_BaseObject, w_type) track_reference(space, obj, w_obj) - if w_type is not space.gettypefor(self.W_BaseObject): - state = space.fromcache(RefcountState) - state.set_lifeline(w_obj, obj) return w_obj typedescr_cache = {} @@ -111,7 +115,7 @@ def init_pyobject(space): from pypy.module.cpyext.object import PyObject_dealloc # typedescr for the 'object' type - make_typedescr(space.w_object.instancetypedef, + make_typedescr(space.w_object.layout.typedef, dealloc=PyObject_dealloc) # almost all types, which should better inherit from object. make_typedescr(None) @@ -134,104 +138,6 @@ #________________________________________________________ # refcounted object support -class RefcountState: - def __init__(self, space): - self.space = space - self.py_objects_w2r = {} # { w_obj -> raw PyObject } - self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } - - self.lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) - - self.borrow_mapping = {None: {}} - # { w_container -> { w_containee -> None } } - # the None entry manages references borrowed during a call to - # generic_cpy_call() - - # For tests - self.non_heaptypes_w = [] - - def _cleanup_(self): - assert self.borrow_mapping == {None: {}} - self.py_objects_r2w.clear() # is not valid anymore after translation - - def init_r2w_from_w2r(self): - """Rebuilds the dict py_objects_r2w on startup""" - for w_obj, obj in self.py_objects_w2r.items(): - ptr = rffi.cast(ADDR, obj) - self.py_objects_r2w[ptr] = w_obj - - def print_refcounts(self): - print "REFCOUNTS" - for w_obj, obj in self.py_objects_w2r.items(): - print "%r: %i" % (w_obj, obj.c_ob_refcnt) - - def get_from_lifeline(self, w_obj): - lifeline = self.lifeline_dict.get(w_obj) - if lifeline is not None: # make old PyObject ready for use in C code - py_obj = lifeline.pyo - assert py_obj.c_ob_refcnt == 0 - return py_obj - else: - return lltype.nullptr(PyObject.TO) - - def set_lifeline(self, w_obj, py_obj): - self.lifeline_dict.set(w_obj, - PyOLifeline(self.space, py_obj)) - - def make_borrowed(self, w_container, w_borrowed): - """ - Create a borrowed reference, which will live as long as the container - has a living reference (as a PyObject!) - """ - ref = make_ref(self.space, w_borrowed) - obj_ptr = rffi.cast(ADDR, ref) - - borrowees = self.borrow_mapping.setdefault(w_container, {}) - if w_borrowed in borrowees: - Py_DecRef(self.space, w_borrowed) # cancel incref from make_ref() - else: - borrowees[w_borrowed] = None - - return ref - - def reset_borrowed_references(self): - "Used in tests" - for w_container, w_borrowed in self.borrow_mapping.items(): - Py_DecRef(self.space, w_borrowed) - self.borrow_mapping = {None: {}} - - def delete_borrower(self, w_obj): - """ - Called when a potential container for borrowed references has lost its - last reference. Removes the borrowed references it contains. - """ - if w_obj in self.borrow_mapping: # move to lifeline __del__ - for w_containee in self.borrow_mapping[w_obj]: - self.forget_borrowee(w_containee) - del self.borrow_mapping[w_obj] - - def swap_borrow_container(self, container): - """switch the current default contained with the given one.""" - if container is None: - old_container = self.borrow_mapping[None] - self.borrow_mapping[None] = {} - return old_container - else: - old_container = self.borrow_mapping[None] - self.borrow_mapping[None] = container - for w_containee in old_container: - self.forget_borrowee(w_containee) - - def forget_borrowee(self, w_obj): - "De-register an object from the list of borrowed references" - ref = self.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) - if not ref: - if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone!" - return - - Py_DecRef(self.space, ref) - class InvalidPointerException(Exception): pass @@ -249,55 +155,37 @@ def create_ref(space, w_obj, itemcount=0): """ Allocates a PyObject, and fills its fields with info from the given - intepreter object. + interpreter object. """ - state = space.fromcache(RefcountState) w_type = space.type(w_obj) - if w_type.is_cpytype(): - py_obj = state.get_from_lifeline(w_obj) - if py_obj: - Py_IncRef(space, py_obj) - return py_obj - typedescr = get_typedescr(w_obj.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - if w_type.is_cpytype(): - state.set_lifeline(w_obj, py_obj) + track_reference(space, py_obj, w_obj) + # + # py_obj.c_ob_refcnt should be exactly REFCNT_FROM_PYPY + 1 here, + # and we want only REFCNT_FROM_PYPY, i.e. only count as attached + # to the W_Root but not with any reference from the py_obj side. + assert py_obj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY + py_obj.c_ob_refcnt -= 1 + # typedescr.attach(space, py_obj, w_obj) return py_obj -def track_reference(space, py_obj, w_obj, replace=False): +def track_reference(space, py_obj, w_obj): """ Ties together a PyObject and an interpreter object. + The PyObject's refcnt is increased by REFCNT_FROM_PYPY. + The reference in 'py_obj' is not stolen! Remember to Py_DecRef() + it is you need to. """ # XXX looks like a PyObject_GC_TRACK - ptr = rffi.cast(ADDR, py_obj) - state = space.fromcache(RefcountState) + assert py_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY + py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY if DEBUG_REFCOUNT: debug_refcount("MAKREF", py_obj, w_obj) - if not replace: - assert w_obj not in state.py_objects_w2r - assert ptr not in state.py_objects_r2w - state.py_objects_w2r[w_obj] = py_obj - if ptr: # init_typeobject() bootstraps with NULL references - state.py_objects_r2w[ptr] = w_obj - -def make_ref(space, w_obj): - """ - Returns a new reference to an intepreter object. - """ - if w_obj is None: - return lltype.nullptr(PyObject.TO) - assert isinstance(w_obj, W_Root) - state = space.fromcache(RefcountState) - try: - py_obj = state.py_objects_w2r[w_obj] - except KeyError: - py_obj = create_ref(space, w_obj) - track_reference(space, py_obj, w_obj) - else: - Py_IncRef(space, py_obj) - return py_obj + assert w_obj + assert py_obj + rawrefcount.create_link_pypy(w_obj, py_obj) def from_ref(space, ref): @@ -305,16 +193,12 @@ Finds the interpreter object corresponding to the given reference. If the object is not yet realized (see bytesobject.py), creates it. """ - assert lltype.typeOf(ref) == PyObject + assert is_pyobj(ref) if not ref: return None - state = space.fromcache(RefcountState) - ptr = rffi.cast(ADDR, ref) - - try: - return state.py_objects_r2w[ptr] - except KeyError: - pass + w_obj = rawrefcount.to_obj(W_Root, ref) + if w_obj is not None: + return w_obj # This reference is not yet a real interpreter object. # Realize it. @@ -323,126 +207,135 @@ raise InvalidPointerException(str(ref)) w_type = from_ref(space, ref_type) assert isinstance(w_type, W_TypeObject) - return get_typedescr(w_type.instancetypedef).realize(space, ref) + return get_typedescr(w_type.layout.typedef).realize(space, ref) -# XXX Optimize these functions and put them into macro definitions -@cpython_api([PyObject], lltype.Void) -def Py_DecRef(space, obj): - if not obj: - return - assert lltype.typeOf(obj) == PyObject +def debug_collect(): + rawrefcount._collect() - obj.c_ob_refcnt -= 1 - if DEBUG_REFCOUNT: - debug_refcount("DECREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) - if obj.c_ob_refcnt == 0: - state = space.fromcache(RefcountState) - ptr = rffi.cast(ADDR, obj) - if ptr not in state.py_objects_r2w: - # this is a half-allocated object, lets call the deallocator - # without modifying the r2w/w2r dicts - _Py_Dealloc(space, obj) - else: - w_obj = state.py_objects_r2w[ptr] - del state.py_objects_r2w[ptr] - w_type = space.type(w_obj) - if not w_type.is_cpytype(): + +def as_pyobj(space, w_obj): + """ + Returns a 'PyObject *' representing the given intepreter object. + This doesn't give a new reference, but the returned 'PyObject *' + is valid at least as long as 'w_obj' is. **To be safe, you should + use keepalive_until_here(w_obj) some time later.** In case of + doubt, use the safer make_ref(). + """ + if w_obj is not None: + assert not is_pyobj(w_obj) + py_obj = rawrefcount.from_obj(PyObject, w_obj) + if not py_obj: + py_obj = create_ref(space, w_obj) + return py_obj + else: + return lltype.nullptr(PyObject.TO) +as_pyobj._always_inline_ = 'try' +INTERPLEVEL_API['as_pyobj'] = as_pyobj + +def pyobj_has_w_obj(pyobj): + return rawrefcount.to_obj(W_Root, pyobj) is not None +INTERPLEVEL_API['pyobj_has_w_obj'] = staticmethod(pyobj_has_w_obj) + + +def is_pyobj(x): + if x is None or isinstance(x, W_Root): + return False + elif is_PyObject(lltype.typeOf(x)): + return True + else: + raise TypeError(repr(type(x))) +INTERPLEVEL_API['is_pyobj'] = staticmethod(is_pyobj) + +class Entry(ExtRegistryEntry): + _about_ = is_pyobj + def compute_result_annotation(self, s_x): + from rpython.rtyper.llannotation import SomePtr + return self.bookkeeper.immutablevalue(isinstance(s_x, SomePtr)) + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputconst(lltype.Bool, hop.s_result.const) + +@specialize.ll() +def make_ref(space, obj): + """Increment the reference counter of the PyObject and return it. + Can be called with either a PyObject or a W_Root. + """ + if is_pyobj(obj): + pyobj = rffi.cast(PyObject, obj) + else: + pyobj = as_pyobj(space, obj) + if pyobj: + assert pyobj.c_ob_refcnt > 0 + pyobj.c_ob_refcnt += 1 + if not is_pyobj(obj): + keepalive_until_here(obj) + return pyobj +INTERPLEVEL_API['make_ref'] = make_ref + + +@specialize.ll() +def get_w_obj_and_decref(space, obj): + """Decrement the reference counter of the PyObject and return the + corresponding W_Root object (so the reference count is at least + REFCNT_FROM_PYPY and cannot be zero). Can be called with either + a PyObject or a W_Root. + """ + if is_pyobj(obj): + pyobj = rffi.cast(PyObject, obj) + w_obj = from_ref(space, pyobj) + else: + w_obj = obj + pyobj = as_pyobj(space, w_obj) + if pyobj: + pyobj.c_ob_refcnt -= 1 + assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY + keepalive_until_here(w_obj) + return w_obj +INTERPLEVEL_API['get_w_obj_and_decref'] = get_w_obj_and_decref + + +@specialize.ll() +def incref(space, obj): + make_ref(space, obj) +INTERPLEVEL_API['incref'] = incref + +@specialize.ll() +def decref(space, obj): + if is_pyobj(obj): + obj = rffi.cast(PyObject, obj) + if obj: + assert obj.c_ob_refcnt > 0 + obj.c_ob_refcnt -= 1 + if obj.c_ob_refcnt == 0: _Py_Dealloc(space, obj) - del state.py_objects_w2r[w_obj] - # if the object was a container for borrowed references - state.delete_borrower(w_obj) else: - if not we_are_translated() and obj.c_ob_refcnt < 0: - message = "Negative refcount for obj %s with type %s" % ( - obj, rffi.charp2str(obj.c_ob_type.c_tp_name)) - print >>sys.stderr, message - assert False, message + get_w_obj_and_decref(space, obj) +INTERPLEVEL_API['decref'] = decref + @cpython_api([PyObject], lltype.Void) def Py_IncRef(space, obj): - if not obj: - return - obj.c_ob_refcnt += 1 - assert obj.c_ob_refcnt > 0 - if DEBUG_REFCOUNT: - debug_refcount("INCREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) + incref(space, obj) + +@cpython_api([PyObject], lltype.Void) +def Py_DecRef(space, obj): + decref(space, obj) @cpython_api([PyObject], lltype.Void) def _Py_NewReference(space, obj): obj.c_ob_refcnt = 1 w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) assert isinstance(w_type, W_TypeObject) - get_typedescr(w_type.instancetypedef).realize(space, obj) + get_typedescr(w_type.layout.typedef).realize(space, obj) +@cpython_api([PyObject], lltype.Void) def _Py_Dealloc(space, obj): - from pypy.module.cpyext.api import generic_cpy_call_dont_decref + from pypy.module.cpyext.api import generic_cpy_call pto = obj.c_ob_type #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \ # "'s type which is", rffi.charp2str(pto.c_tp_name) - generic_cpy_call_dont_decref(space, pto.c_tp_dealloc, obj) - -#___________________________________________________________ -# Support for "lifelines" -# -# Object structure must stay alive even when not referenced -# by any C code. - -class PyOLifeline(object): - def __init__(self, space, pyo): - self.pyo = pyo - self.space = space - - def __del__(self): - if self.pyo: - assert self.pyo.c_ob_refcnt == 0 - _Py_Dealloc(self.space, self.pyo) - self.pyo = lltype.nullptr(PyObject.TO) - # XXX handle borrowed objects here - -#___________________________________________________________ -# Support for borrowed references - -def make_borrowed_ref(space, w_container, w_borrowed): - """ - Create a borrowed reference, which will live as long as the container - has a living reference (as a PyObject!) - """ - if w_borrowed is None: - return lltype.nullptr(PyObject.TO) - - state = space.fromcache(RefcountState) - return state.make_borrowed(w_container, w_borrowed) - -class Reference: - def __init__(self, pyobj): - assert not isinstance(pyobj, W_Root) - self.pyobj = pyobj - - def get_ref(self, space): - return self.pyobj - - def get_wrapped(self, space): - return from_ref(space, self.pyobj) - -class BorrowPair(Reference): - """ - Delays the creation of a borrowed reference. - """ - def __init__(self, w_container, w_borrowed): - self.w_container = w_container - self.w_borrowed = w_borrowed - - def get_ref(self, space): - return make_borrowed_ref(space, self.w_container, self.w_borrowed) - - def get_wrapped(self, space): - return self.w_borrowed - -def borrow_from(container, borrowed): - return BorrowPair(container, borrowed) - -#___________________________________________________________ + generic_cpy_call(space, pto.c_tp_dealloc, obj) @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL) def _Py_HashPointer(space, ptr): diff --git a/pypy/module/cpyext/pypyintf.py b/pypy/module/cpyext/pypyintf.py deleted file mode 100644 --- a/pypy/module/cpyext/pypyintf.py +++ /dev/null @@ -1,9 +0,0 @@ -from pypy.module.cpyext.api import cpython_api -from pypy.module.cpyext.pyobject import PyObject, borrow_from - - -@cpython_api([PyObject, PyObject], PyObject) -def PyPy_Borrow(space, w_parentobj, w_obj): - """Returns a borrowed reference to 'obj', borrowing from the 'parentobj'. - """ - return borrow_from(w_parentobj, w_obj) diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -3,7 +3,7 @@ PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t, cpython_api, bootstrap_function, cpython_struct, build_type_checkers) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, borrow_from) + PyObject, make_ref, from_ref, Py_DecRef, make_typedescr) from pypy.module.cpyext.frameobject import PyFrameObject from rpython.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t) -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from rpython.rtyper.lltypesystem import rffi, lltype from pypy.objspace.std import listobject, tupleobject @@ -42,15 +42,19 @@ which case o is returned. Use PySequence_Fast_GET_ITEM() to access the members of the result. Returns NULL on failure. If the object is not a sequence, raises TypeError with m as the message text.""" - if (isinstance(w_obj, listobject.W_ListObject) or - isinstance(w_obj, tupleobject.W_TupleObject)): + if isinstance(w_obj, listobject.W_ListObject): + # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM + # XXX how does this interact with CPyListStrategy? + w_obj.ensure_object_strategy() + return w_obj + if isinstance(w_obj, tupleobject.W_TupleObject): return w_obj try: return tupleobject.W_TupleObject(space.fixedview(w_obj)) except OperationError: raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m))) -@cpython_api([PyObject, Py_ssize_t], PyObject) +@cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True) def PySequence_Fast_GET_ITEM(space, w_obj, index): """Return the ith element of o, assuming that o was returned by PySequence_Fast(), o is not NULL, and that i is within bounds. @@ -60,7 +64,7 @@ else: assert isinstance(w_obj, tupleobject.W_TupleObject) w_res = w_obj.wrappeditems[index] - return borrow_from(w_obj, w_res) + return w_res # borrowed ref @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PySequence_Fast_GET_SIZE(space, w_obj): diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py --- a/pypy/module/cpyext/setobject.py +++ b/pypy/module/cpyext/setobject.py @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - borrow_from, make_ref, from_ref) + make_ref, from_ref) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.setobject import W_SetObject, newset diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -31,17 +31,16 @@ Py_GE = 5 -def check_num_args(space, ob, n): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact, \ - PyTuple_GET_SIZE - if not PyTuple_CheckExact(space, ob): +def check_num_args(space, w_ob, n): + from pypy.module.cpyext.tupleobject import PyTuple_CheckExact + if not PyTuple_CheckExact(space, w_ob): raise OperationError(space.w_SystemError, space.wrap("PyArg_UnpackTuple() argument list is not a tuple")) - if n == PyTuple_GET_SIZE(space, ob): + if n == space.len_w(w_ob): return raise oefmt(space.w_TypeError, _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit