Author: Jeremy Thurgood <fir...@gmail.com> Branch: unrecursive-opt Changeset: r80154:c3e4bdc5e479 Date: 2015-10-13 10:45 +0200 http://bitbucket.org/pypy/pypy/changeset/c3e4bdc5e479/
Log: Merge default. diff too long, truncating to 2000 out of 8776 lines diff --git a/dotviewer/graphclient.py b/dotviewer/graphclient.py --- a/dotviewer/graphclient.py +++ b/dotviewer/graphclient.py @@ -127,16 +127,8 @@ return spawn_graphserver_handler((host, port)) def spawn_local_handler(): - if hasattr(sys, 'pypy_objspaceclass'): - # if 'python' is actually PyPy, e.g. in a virtualenv, then - # try hard to find a real CPython - try: - python = subprocess.check_output( - 'env -i $SHELL -l -c "which python"', shell=True).strip() - except subprocess.CalledProcessError: - # did not work, fall back to 'python' - python = 'python' - else: + python = os.getenv('PYPY_PYGAME_PYTHON') + if not python: python = sys.executable args = [python, '-u', GRAPHSERVER, '--stdio'] p = subprocess.Popen(args, diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -158,7 +158,7 @@ RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'), RegrTest('test_codeop.py', core=True), RegrTest('test_coding.py', core=True), - RegrTest('test_coercion.py', core=True), + RegrTest('test_coercion.py', core=True, usemodules='struct'), RegrTest('test_collections.py', usemodules='binascii struct'), RegrTest('test_colorsys.py'), RegrTest('test_commands.py'), diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -310,6 +310,22 @@ """ return self._backend.from_buffer(self.BCharA, python_buffer) + def memmove(self, dest, src, n): + """ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. + + Like the C function memmove(), the memory areas may overlap; + apart from that it behaves like the C function memcpy(). + + 'src' can be any cdata ptr or array, or any Python buffer object. + 'dest' can be any cdata ptr or array, or a writable Python buffer + object. The size to copy, 'n', is always measured in bytes. + + Unlike other methods, this one supports all Python buffer including + byte strings and bytearrays---but it still does not support + non-contiguous buffers. + """ + return self._backend.memmove(dest, src, n) + def callback(self, cdecl, python_callable=None, error=None, onerror=None): """Return a callback object or a decorator making such a callback object. 'cdecl' must name a C function pointer type. diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -26,6 +26,9 @@ _r_words = re.compile(r"\w+|\S") _parser_cache = None _r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE) +_r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b") +_r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b") +_r_cdecl = re.compile(r"\b__cdecl\b") def _get_parser(): global _parser_cache @@ -44,6 +47,14 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + # BIG HACK: replace WINAPI or __stdcall with "volatile const". + # It doesn't make sense for the return type of a function to be + # "volatile volatile const", so we abuse it to detect __stdcall... + # Hack number 2 is that "int(volatile *fptr)();" is not valid C + # syntax, so we place the "volatile" before the opening parenthesis. + csource = _r_stdcall2.sub(' volatile volatile const(', csource) + csource = _r_stdcall1.sub(' volatile volatile const ', csource) + csource = _r_cdecl.sub(' ', csource) # Replace "[...]" with "[__dotdotdotarray__]" csource = _r_partial_array.sub('[__dotdotdotarray__]', csource) # Replace "...}" with "__dotdotdotNUM__}". This construction should @@ -449,7 +460,14 @@ if not ellipsis and args == [model.void_type]: args = [] result, quals = self._get_type_and_quals(typenode.type) - return model.RawFunctionType(tuple(args), result, ellipsis) + # the 'quals' on the result type are ignored. HACK: we absure them + # to detect __stdcall functions: we textually replace "__stdcall" + # with "volatile volatile const" above. + abi = None + if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway + if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']: + abi = '__stdcall' + return model.RawFunctionType(tuple(args), result, ellipsis, abi) def _as_func_arg(self, type, quals): if isinstance(type, model.ArrayType): diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -1,4 +1,4 @@ -import types +import types, sys import weakref from .lock import allocate_lock @@ -193,18 +193,21 @@ class BaseFunctionType(BaseType): - _attrs_ = ('args', 'result', 'ellipsis') + _attrs_ = ('args', 'result', 'ellipsis', 'abi') - def __init__(self, args, result, ellipsis): + def __init__(self, args, result, ellipsis, abi=None): self.args = args self.result = result self.ellipsis = ellipsis + self.abi = abi # reprargs = [arg._get_c_name() for arg in self.args] if self.ellipsis: reprargs.append('...') reprargs = reprargs or ['void'] replace_with = self._base_pattern % (', '.join(reprargs),) + if abi is not None: + replace_with = replace_with[:1] + abi + ' ' + replace_with[1:] self.c_name_with_marker = ( self.result.c_name_with_marker.replace('&', replace_with)) @@ -222,7 +225,7 @@ "type, not a pointer-to-function type" % (self,)) def as_function_pointer(self): - return FunctionPtrType(self.args, self.result, self.ellipsis) + return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi) class FunctionPtrType(BaseFunctionType): @@ -233,11 +236,18 @@ args = [] for tp in self.args: args.append(tp.get_cached_btype(ffi, finishlist)) + abi_args = () + if self.abi == "__stdcall": + if not self.ellipsis: # __stdcall ignored for variadic funcs + try: + abi_args = (ffi._backend.FFI_STDCALL,) + except AttributeError: + pass return global_cache(self, ffi, 'new_function_type', - tuple(args), result, self.ellipsis) + tuple(args), result, self.ellipsis, *abi_args) def as_raw_function(self): - return RawFunctionType(self.args, self.result, self.ellipsis) + return RawFunctionType(self.args, self.result, self.ellipsis, self.abi) class PointerType(BaseType): diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h --- a/lib_pypy/cffi/parse_c_type.h +++ b/lib_pypy/cffi/parse_c_type.h @@ -5,7 +5,7 @@ #define _CFFI_OP(opcode, arg) (_cffi_opcode_t)(opcode | (((uintptr_t)(arg)) << 8)) #define _CFFI_GETOP(cffi_opcode) ((unsigned char)(uintptr_t)cffi_opcode) -#define _CFFI_GETARG(cffi_opcode) (((uintptr_t)cffi_opcode) >> 8) +#define _CFFI_GETARG(cffi_opcode) (((intptr_t)cffi_opcode) >> 8) #define _CFFI_OP_PRIMITIVE 1 #define _CFFI_OP_POINTER 3 diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -607,7 +607,11 @@ call_arguments.append('x%d' % i) repr_arguments = ', '.join(arguments) repr_arguments = repr_arguments or 'void' - name_and_arguments = '_cffi_d_%s(%s)' % (name, repr_arguments) + if tp.abi: + abi = tp.abi + ' ' + else: + abi = '' + name_and_arguments = '%s_cffi_d_%s(%s)' % (abi, name, repr_arguments) prnt('static %s' % (tp.result.get_c_name(name_and_arguments),)) prnt('{') call_arguments = ', '.join(call_arguments) @@ -710,7 +714,8 @@ if difference: repr_arguments = ', '.join(arguments) repr_arguments = repr_arguments or 'void' - name_and_arguments = '_cffi_f_%s(%s)' % (name, repr_arguments) + name_and_arguments = '%s_cffi_f_%s(%s)' % (abi, name, + repr_arguments) prnt('static %s' % (tp_result.get_c_name(name_and_arguments),)) prnt('{') if result_decl: @@ -1135,7 +1140,13 @@ else: self.cffi_types[index] = CffiOp(OP_NOOP, realindex) index += 1 - self.cffi_types[index] = CffiOp(OP_FUNCTION_END, int(tp.ellipsis)) + flags = int(tp.ellipsis) + if tp.abi is not None: + if tp.abi == '__stdcall': + flags |= 2 + else: + raise NotImplementedError("abi=%r" % (tp.abi,)) + self.cffi_types[index] = CffiOp(OP_FUNCTION_END, flags) def _emit_bytecode_PointerType(self, tp, index): self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype]) diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -159,7 +159,11 @@ arglist = ', '.join(arglist) or 'void' wrappername = '_cffi_f_%s' % name self.export_symbols.append(wrappername) - funcdecl = ' %s(%s)' % (wrappername, arglist) + if tp.abi: + abi = tp.abi + ' ' + else: + abi = '' + funcdecl = ' %s%s(%s)' % (abi, wrappername, arglist) context = 'result of %s' % name prnt(tpresult.get_c_name(funcdecl, context)) prnt('{') diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst --- a/pypy/doc/extending.rst +++ b/pypy/doc/extending.rst @@ -83,7 +83,7 @@ RPython Mixed Modules -===================== +--------------------- This is the internal way to write built-in extension modules in PyPy. It cannot be used by any 3rd-party module: the extension modules are diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -43,3 +43,34 @@ .. branch: numpy-ctypes Add support for ndarray.ctypes property. + +.. branch: share-guard-info + +Share guard resume data between consecutive guards that have only +pure operations and guards in between. + +.. branch: issue-2148 + +Fix performance regression on operations mixing numpy scalars and Python +floats, cf. issue #2148. + +.. branch: cffi-stdcall +Win32: support '__stdcall' in CFFI. + +.. branch: callfamily + +Refactorings of annotation and rtyping of function calls. + +.. branch: fortran-order + +Allow creation of fortran-ordered ndarrays + +.. branch: type_system-cleanup + +Remove some remnants of the old ootypesystem vs lltypesystem dichotomy. + +.. branch: cffi-handle-lifetime + +ffi.new_handle() returns handles that work more like CPython's: they +remain valid as long as the target exists (unlike the previous +version, where handles become invalid *before* the __del__ is called). diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1058,6 +1058,14 @@ args = Arguments.frompacked(self, w_args, w_kwds) return self.call_args(w_callable, args) + def _try_fetch_pycode(self, w_func): + from pypy.interpreter.function import Function, Method + if isinstance(w_func, Method): + w_func = w_func.w_function + if isinstance(w_func, Function): + return w_func.code + return None + def call_function(self, w_func, *args_w): nargs = len(args_w) # used for pruning funccall versions if not self.config.objspace.disable_call_speedhacks and nargs < 5: diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -1,9 +1,16 @@ import sys from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib import rdynload +from rpython.rlib import rdynload, clibffi VERSION = "1.3.0" +FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI +try: + FFI_STDCALL = clibffi.FFI_STDCALL + has_stdcall = True +except AttributeError: + has_stdcall = False + class Module(MixedModule): @@ -40,12 +47,13 @@ 'string': 'func.string', 'buffer': 'cbuffer.buffer', + 'memmove': 'func.memmove', 'get_errno': 'cerrno.get_errno', 'set_errno': 'cerrno.set_errno', - 'FFI_DEFAULT_ABI': 'ctypefunc._get_abi(space, "FFI_DEFAULT_ABI")', - 'FFI_CDECL': 'ctypefunc._get_abi(space,"FFI_DEFAULT_ABI")',#win32 name + 'FFI_DEFAULT_ABI': 'space.wrap(%d)' % FFI_DEFAULT_ABI, + 'FFI_CDECL': 'space.wrap(%d)' % FFI_DEFAULT_ABI, # win32 name # CFFI 1.0 'FFI': 'ffi_obj.W_FFIObject', @@ -53,6 +61,9 @@ if sys.platform == 'win32': interpleveldefs['getwinerror'] = 'cerrno.getwinerror' + if has_stdcall: + interpleveldefs['FFI_STDCALL'] = 'space.wrap(%d)' % FFI_STDCALL + def get_dict_rtld_constants(): found = {} diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -1,11 +1,11 @@ """ Callbacks. """ -import sys, os +import sys, os, py -from rpython.rlib import clibffi, rweakref, jit, jit_libffi -from rpython.rlib.objectmodel import compute_unique_id, keepalive_until_here -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import clibffi, jit, jit_libffi, rgc, objectmodel +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt from pypy.module._cffi_backend import cerrno, misc @@ -19,6 +19,23 @@ # ____________________________________________________________ +@jit.dont_look_inside +def make_callback(space, ctype, w_callable, w_error, w_onerror): + # Allocate a callback as a nonmovable W_CDataCallback instance, which + # we can cast to a plain VOIDP. As long as the object is not freed, + # we can cast the VOIDP back to a W_CDataCallback in reveal_callback(). + cdata = objectmodel.instantiate(W_CDataCallback, nonmovable=True) + gcref = rgc.cast_instance_to_gcref(cdata) + raw_cdata = rgc.hide_nonmovable_gcref(gcref) + cdata.__init__(space, ctype, w_callable, w_error, w_onerror, raw_cdata) + return cdata + +def reveal_callback(raw_ptr): + addr = rffi.cast(llmemory.Address, raw_ptr) + gcref = rgc.reveal_gcref(addr) + return rgc.try_cast_gcref_to_instance(W_CDataCallback, gcref) + + class Closure(object): """This small class is here to have a __del__ outside any cycle.""" @@ -34,10 +51,11 @@ class W_CDataCallback(W_CData): - #_immutable_fields_ = ... + _immutable_fields_ = ['key_pycode'] w_onerror = None - def __init__(self, space, ctype, w_callable, w_error, w_onerror): + def __init__(self, space, ctype, w_callable, w_error, w_onerror, + raw_cdata): raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc()) self._closure = Closure(raw_closure) W_CData.__init__(self, space, raw_closure, ctype) @@ -46,6 +64,7 @@ raise oefmt(space.w_TypeError, "expected a callable object, not %T", w_callable) self.w_callable = w_callable + self.key_pycode = space._try_fetch_pycode(w_callable) if not space.is_none(w_onerror): if not space.is_true(space.callable(w_onerror)): raise oefmt(space.w_TypeError, @@ -64,8 +83,12 @@ convert_from_object_fficallback(fresult, self._closure.ll_error, w_error) # - self.unique_id = compute_unique_id(self) - global_callback_mapping.set(self.unique_id, self) + # We must setup the GIL here, in case the callback is invoked in + # some other non-Pythonic thread. This is the same as cffi on + # CPython. + if space.config.translation.thread: + from pypy.module.thread.os_thread import setup_threads + setup_threads(space) # cif_descr = self.getfunctype().cif_descr if not cif_descr: @@ -74,20 +97,13 @@ "return type or with '...'", self.getfunctype().name) with self as ptr: closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr) - unique_id = rffi.cast(rffi.VOIDP, self.unique_id) + unique_id = rffi.cast(rffi.VOIDP, raw_cdata) res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif, invoke_callback, unique_id) if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: raise OperationError(space.w_SystemError, space.wrap("libffi failed to build this callback")) - # - # We must setup the GIL here, in case the callback is invoked in - # some other non-Pythonic thread. This is the same as cffi on - # CPython. - if space.config.translation.thread: - from pypy.module.thread.os_thread import setup_threads - setup_threads(space) def _repr_extra(self): space = self.space @@ -105,6 +121,7 @@ def invoke(self, ll_args): space = self.space ctype = self.getfunctype() + ctype = jit.promote(ctype) args_w = [] for i, farg in enumerate(ctype.fargs): ll_arg = rffi.cast(rffi.CCHARP, ll_args[i]) @@ -127,9 +144,6 @@ keepalive_until_here(self) # to keep self._closure.ll_error alive -global_callback_mapping = rweakref.RWeakValueDictionary(int, W_CDataCallback) - - def convert_from_object_fficallback(fresult, ll_res, w_res): space = fresult.space small_result = fresult.size < SIZE_OF_FFI_ARG @@ -178,7 +192,8 @@ @jit.dont_look_inside -def _handle_applevel_exception(space, callback, e, ll_res, extra_line): +def _handle_applevel_exception(callback, e, ll_res, extra_line): + space = callback.space callback.write_error_return_value(ll_res) if callback.w_onerror is None: callback.print_error(e, extra_line) @@ -199,19 +214,36 @@ extra_line="\nDuring the call to 'onerror', " "another exception occurred:\n\n") +def get_printable_location(key_pycode): + if key_pycode is None: + return 'cffi_callback <?>' + return 'cffi_callback ' + key_pycode.get_repr() -@jit.jit_callback("CFFI") +jitdriver = jit.JitDriver(name='cffi_callback', + greens=['callback.key_pycode'], + reds=['ll_res', 'll_args', 'callback'], + get_printable_location=get_printable_location) + +def py_invoke_callback(callback, ll_res, ll_args): + jitdriver.jit_merge_point(callback=callback, ll_res=ll_res, ll_args=ll_args) + extra_line = '' + try: + w_res = callback.invoke(ll_args) + extra_line = "Trying to convert the result back to C:\n" + callback.convert_result(ll_res, w_res) + except OperationError, e: + _handle_applevel_exception(callback, e, ll_res, extra_line) + def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care ll_args - rffi.VOIDPP - pointer to array of pointers to args - ll_restype - rffi.VOIDP - pointer to result + ll_res - rffi.VOIDP - pointer to result ll_userdata - a special structure which holds necessary information (what the real callback is for example), casted to VOIDP """ ll_res = rffi.cast(rffi.CCHARP, ll_res) - unique_id = rffi.cast(lltype.Signed, ll_userdata) - callback = global_callback_mapping.get(unique_id) + callback = reveal_callback(ll_userdata) if callback is None: # oups! try: @@ -224,17 +256,11 @@ misc._raw_memclear(ll_res, SIZE_OF_FFI_ARG) return # + space = callback.space must_leave = False - space = callback.space try: must_leave = space.threadlocals.try_enter_thread(space) - extra_line = '' - try: - w_res = callback.invoke(ll_args) - extra_line = "Trying to convert the result back to C:\n" - callback.convert_result(ll_res, w_res) - except OperationError, e: - _handle_applevel_exception(space, callback, e, ll_res, extra_line) + py_invoke_callback(callback, ll_res, ll_args) # except Exception, e: # oups! last-level attempt to recover. diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -18,6 +18,7 @@ _attrs_ = ['ctptr'] _immutable_fields_ = ['ctptr'] kind = "array" + is_nonfunc_pointer_or_array = True def __init__(self, space, ctptr, length, arraysize, extra): W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0, diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -12,6 +12,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt +from pypy.module import _cffi_backend from pypy.module._cffi_backend import ctypearray, cdataobj, cerrno from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend.ctypeptr import W_CTypePtrBase, W_CTypePointer @@ -23,20 +24,22 @@ class W_CTypeFunc(W_CTypePtrBase): - _attrs_ = ['fargs', 'ellipsis', 'cif_descr'] - _immutable_fields_ = ['fargs[*]', 'ellipsis', 'cif_descr'] + _attrs_ = ['fargs', 'ellipsis', 'abi', 'cif_descr'] + _immutable_fields_ = ['fargs[*]', 'ellipsis', 'abi', 'cif_descr'] kind = "function" cif_descr = lltype.nullptr(CIF_DESCRIPTION) - def __init__(self, space, fargs, fresult, ellipsis): + def __init__(self, space, fargs, fresult, ellipsis, + abi=_cffi_backend.FFI_DEFAULT_ABI): assert isinstance(ellipsis, bool) - extra = self._compute_extra_text(fargs, fresult, ellipsis) + extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi) size = rffi.sizeof(rffi.VOIDP) - W_CTypePtrBase.__init__(self, space, size, extra, 2, fresult, + W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult, could_cast_anything=False) self.fargs = fargs self.ellipsis = ellipsis + self.abi = abi # fresult is stored in self.ctitem if not ellipsis: @@ -44,7 +47,7 @@ # at all. The cif is computed on every call from the actual # types passed in. For all other functions, the cif_descr # is computed here. - builder = CifDescrBuilder(fargs, fresult) + builder = CifDescrBuilder(fargs, fresult, abi) try: builder.rawallocate(self) except OperationError, e: @@ -76,7 +79,7 @@ ctypefunc.fargs = fvarargs ctypefunc.ctitem = self.ctitem #ctypefunc.cif_descr = NULL --- already provided as the default - CifDescrBuilder(fvarargs, self.ctitem).rawallocate(ctypefunc) + CifDescrBuilder(fvarargs, self.ctitem, self.abi).rawallocate(ctypefunc) return ctypefunc @rgc.must_be_light_finalizer @@ -84,8 +87,13 @@ if self.cif_descr: lltype.free(self.cif_descr, flavor='raw') - def _compute_extra_text(self, fargs, fresult, ellipsis): + def _compute_extra_text(self, fargs, fresult, ellipsis, abi): + from pypy.module._cffi_backend import newtype argnames = ['(*)('] + xpos = 2 + if _cffi_backend.has_stdcall and abi == _cffi_backend.FFI_STDCALL: + argnames[0] = '(__stdcall *)(' + xpos += len('__stdcall ') for i, farg in enumerate(fargs): if i > 0: argnames.append(', ') @@ -95,7 +103,7 @@ argnames.append(', ') argnames.append('...') argnames.append(')') - return ''.join(argnames) + return ''.join(argnames), xpos def _fget(self, attrchar): if attrchar == 'a': # args @@ -106,7 +114,7 @@ if attrchar == 'E': # ellipsis return self.space.wrap(self.ellipsis) if attrchar == 'A': # abi - return self.space.wrap(clibffi.FFI_DEFAULT_ABI) # XXX + return self.space.wrap(self.abi) return W_CTypePtrBase._fget(self, attrchar) def call(self, funcaddr, args_w): @@ -181,11 +189,6 @@ def set_mustfree_flag(data, flag): rffi.ptradd(data, -1)[0] = chr(flag) -def _get_abi(space, name): - abi = getattr(clibffi, name) - assert isinstance(abi, int) - return space.wrap(abi) - # ____________________________________________________________ @@ -260,9 +263,10 @@ class CifDescrBuilder(object): rawmem = lltype.nullptr(rffi.CCHARP.TO) - def __init__(self, fargs, fresult): + def __init__(self, fargs, fresult, fabi): self.fargs = fargs self.fresult = fresult + self.fabi = fabi def fb_alloc(self, size): size = llmemory.raw_malloc_usage(size) @@ -421,7 +425,7 @@ cif_descr.exchange_size = exchange_offset def fb_extra_fields(self, cif_descr): - cif_descr.abi = clibffi.FFI_DEFAULT_ABI # XXX + cif_descr.abi = self.fabi cif_descr.nargs = len(self.fargs) cif_descr.rtype = self.rtype cif_descr.atypes = self.atypes diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -21,6 +21,7 @@ cast_anything = False is_primitive_integer = False + is_nonfunc_pointer_or_array = False kind = "?" def __init__(self, space, size, name, name_position): @@ -143,7 +144,7 @@ # obscure hack when untranslated, maybe, approximate, don't use if isinstance(align, llmemory.FieldOffset): align = rffi.sizeof(align.TYPE.y) - if (1 << (8*align-2)) > sys.maxint: + if sys.platform != 'win32' and (1 << (8*align-2)) > sys.maxint: align /= 2 else: # a different hack when translated, to avoid seeing constants diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -172,6 +172,7 @@ _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr'] kind = "pointer" cache_array_type = None + is_nonfunc_pointer_or_array = True def __init__(self, space, ctitem): from pypy.module._cffi_backend import ctypearray diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -294,9 +294,9 @@ CONSIDER_FN_AS_FNPTR) space = self.space if not space.is_none(w_python_callable): - return ccallback.W_CDataCallback(space, w_ctype, - w_python_callable, w_error, - w_onerror) + return ccallback.make_callback(space, w_ctype, + w_python_callable, w_error, + w_onerror) else: # decorator mode: returns a single-argument function return space.appexec([w_ctype, w_error, w_onerror], @@ -391,6 +391,25 @@ return cerrno.getwinerror(self.space, code) + @unwrap_spec(n=int) + def descr_memmove(self, w_dest, w_src, n): + """\ +ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. + +Like the C function memmove(), the memory areas may overlap; +apart from that it behaves like the C function memcpy(). + +'src' can be any cdata ptr or array, or any Python buffer object. +'dest' can be any cdata ptr or array, or a writable Python buffer +object. The size to copy, 'n', is always measured in bytes. + +Unlike other methods, this one supports all Python buffer including +byte strings and bytearrays---but it still does not support +non-contiguous buffers.""" + # + return func.memmove(self.space, w_dest, w_src, n) + + @unwrap_spec(w_init=WrappedDefault(None)) def descr_new(self, w_arg, w_init): """\ @@ -623,6 +642,7 @@ gc = interp2app(W_FFIObject.descr_gc), getctype = interp2app(W_FFIObject.descr_getctype), integer_const = interp2app(W_FFIObject.descr_integer_const), + memmove = interp2app(W_FFIObject.descr_memmove), new = interp2app(W_FFIObject.descr_new), new_allocator = interp2app(W_FFIObject.descr_new_allocator), new_handle = interp2app(W_FFIObject.descr_new_handle), diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -1,3 +1,8 @@ +from rpython.rtyper.annlowlevel import llstr +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw +from rpython.rlib.objectmodel import keepalive_until_here + from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec, WrappedDefault from pypy.module._cffi_backend import ctypeobj, cdataobj, allocator @@ -19,8 +24,8 @@ @unwrap_spec(w_ctype=ctypeobj.W_CType) def callback(space, w_ctype, w_callable, w_error=None, w_onerror=None): - from pypy.module._cffi_backend.ccallback import W_CDataCallback - return W_CDataCallback(space, w_ctype, w_callable, w_error, w_onerror) + from pypy.module._cffi_backend.ccallback import make_callback + return make_callback(space, w_ctype, w_callable, w_error, w_onerror) # ____________________________________________________________ @@ -79,6 +84,26 @@ # ____________________________________________________________ +def _fetch_as_read_buffer(space, w_x): + # xxx do we really need to implement the same mess as in CPython 2.7 + # w.r.t. buffers and memoryviews?? + try: + buf = space.readbuf_w(w_x) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + buf = space.buffer_w(w_x, space.BUF_SIMPLE) + return buf + +def _fetch_as_write_buffer(space, w_x): + try: + buf = space.writebuf_w(w_x) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + buf = space.buffer_w(w_x, space.BUF_WRITABLE) + return buf + @unwrap_spec(w_ctype=ctypeobj.W_CType) def from_buffer(space, w_ctype, w_x): from pypy.module._cffi_backend import ctypearray, ctypeprim @@ -88,14 +113,7 @@ raise oefmt(space.w_TypeError, "needs 'char[]', got '%s'", w_ctype.name) # - # xxx do we really need to implement the same mess as in CPython 2.7 - # w.r.t. buffers and memoryviews?? - try: - buf = space.readbuf_w(w_x) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - buf = space.buffer_w(w_x, space.BUF_SIMPLE) + buf = _fetch_as_read_buffer(space, w_x) try: _cdata = buf.get_raw_address() except ValueError: @@ -106,6 +124,76 @@ # return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x) + +def unsafe_escaping_ptr_for_ptr_or_array(w_cdata): + if not w_cdata.ctype.is_nonfunc_pointer_or_array: + raise oefmt(w_cdata.space.w_TypeError, + "expected a pointer or array ctype, got '%s'", + w_cdata.ctype.name) + return w_cdata.unsafe_escaping_ptr() + +c_memmove = rffi.llexternal('memmove', [rffi.CCHARP, rffi.CCHARP, + rffi.SIZE_T], lltype.Void, + _nowrapper=True) + +@unwrap_spec(n=int) +def memmove(space, w_dest, w_src, n): + if n < 0: + raise oefmt(space.w_ValueError, "negative size") + + # cases... + src_buf = None + src_data = lltype.nullptr(rffi.CCHARP.TO) + if isinstance(w_src, cdataobj.W_CData): + src_data = unsafe_escaping_ptr_for_ptr_or_array(w_src) + src_is_ptr = True + else: + src_buf = _fetch_as_read_buffer(space, w_src) + try: + src_data = src_buf.get_raw_address() + src_is_ptr = True + except ValueError: + src_is_ptr = False + + if src_is_ptr: + src_string = None + else: + if n == src_buf.getlength(): + src_string = src_buf.as_str() + else: + src_string = src_buf.getslice(0, n, 1, n) + + dest_buf = None + dest_data = lltype.nullptr(rffi.CCHARP.TO) + if isinstance(w_dest, cdataobj.W_CData): + dest_data = unsafe_escaping_ptr_for_ptr_or_array(w_dest) + dest_is_ptr = True + else: + dest_buf = _fetch_as_write_buffer(space, w_dest) + try: + dest_data = dest_buf.get_raw_address() + dest_is_ptr = True + except ValueError: + dest_is_ptr = False + + if dest_is_ptr: + if src_is_ptr: + c_memmove(dest_data, src_data, rffi.cast(rffi.SIZE_T, n)) + else: + copy_string_to_raw(llstr(src_string), dest_data, 0, n) + else: + if src_is_ptr: + for i in range(n): + dest_buf.setitem(i, src_data[i]) + else: + for i in range(n): + dest_buf.setitem(i, src_string[i]) + + keepalive_until_here(src_buf) + keepalive_until_here(dest_buf) + keepalive_until_here(w_src) + keepalive_until_here(w_dest) + # ____________________________________________________________ @unwrap_spec(w_cdata=cdataobj.W_CData) diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -1,24 +1,24 @@ +import py from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.baseobjspace import W_Root from pypy.module._cffi_backend import ctypeobj, ctypeptr, cdataobj -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib import rweaklist - - -class CffiHandles(rweaklist.RWeakListMixin): - def __init__(self, space): - self.initialize() - -def get(space): - return space.fromcache(CffiHandles) +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib import rgc, objectmodel, jit # ____________________________________________________________ +@jit.dont_look_inside def _newp_handle(space, w_ctype, w_x): - index = get(space).reserve_next_handle_index() - _cdata = rffi.cast(rffi.CCHARP, index + 1) - new_cdataobj = cdataobj.W_CDataHandle(space, _cdata, w_ctype, w_x) - get(space).store_handle(index, new_cdataobj) + # Allocate a handle as a nonmovable W_CDataHandle instance, which + # we can cast to a plain CCHARP. As long as the object is not freed, + # we can cast the CCHARP back to a W_CDataHandle with reveal_gcref(). + new_cdataobj = objectmodel.instantiate(cdataobj.W_CDataHandle, + nonmovable=True) + gcref = rgc.cast_instance_to_gcref(new_cdataobj) + _cdata = rgc.hide_nonmovable_gcref(gcref) + _cdata = rffi.cast(rffi.CCHARP, _cdata) + cdataobj.W_CDataHandle.__init__(new_cdataobj, space, _cdata, w_ctype, w_x) return new_cdataobj @unwrap_spec(w_ctype=ctypeobj.W_CType) @@ -38,14 +38,17 @@ "expected a 'cdata' object with a 'void *' out of " "new_handle(), got '%s'", ctype.name) with w_cdata as ptr: - index = rffi.cast(lltype.Signed, ptr) - original_cdataobj = get(space).fetch_handle(index - 1) - # - if isinstance(original_cdataobj, cdataobj.W_CDataHandle): - return original_cdataobj.w_keepalive - else: - if index == 0: - msg = "cannot use from_handle() on NULL pointer" - else: - msg = "'void *' value does not correspond to any object" - raise OperationError(space.w_RuntimeError, space.wrap(msg)) + return _reveal(space, ptr) + +@jit.dont_look_inside +def _reveal(space, ptr): + addr = rffi.cast(llmemory.Address, ptr) + gcref = rgc.reveal_gcref(addr) + if not gcref: + raise oefmt(space.w_RuntimeError, + "cannot use from_handle() on NULL pointer") + cd = rgc.try_cast_gcref_to_instance(cdataobj.W_CDataHandle, gcref) + if cd is None: + raise oefmt(space.w_SystemError, + "ffi.from_handle(): dead or bogus object handle") + return cd.w_keepalive diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -4,10 +4,11 @@ from rpython.rlib.objectmodel import specialize, r_dict, compute_identity_hash from rpython.rlib.rarithmetic import ovfcheck, intmask -from rpython.rlib import jit, rweakref +from rpython.rlib import jit, rweakref, clibffi from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.tool import rffi_platform +from pypy.module import _cffi_backend from pypy.module._cffi_backend import (ctypeobj, ctypeprim, ctypeptr, ctypearray, ctypestruct, ctypevoid, ctypeenum) @@ -592,8 +593,9 @@ # ____________________________________________________________ -@unwrap_spec(w_fresult=ctypeobj.W_CType, ellipsis=int) -def new_function_type(space, w_fargs, w_fresult, ellipsis=0): +@unwrap_spec(w_fresult=ctypeobj.W_CType, ellipsis=int, abi=int) +def new_function_type(space, w_fargs, w_fresult, ellipsis=0, + abi=_cffi_backend.FFI_DEFAULT_ABI): fargs = [] for w_farg in space.fixedview(w_fargs): if not isinstance(w_farg, ctypeobj.W_CType): @@ -602,28 +604,28 @@ if isinstance(w_farg, ctypearray.W_CTypeArray): w_farg = w_farg.ctptr fargs.append(w_farg) - return _new_function_type(space, fargs, w_fresult, bool(ellipsis)) + return _new_function_type(space, fargs, w_fresult, bool(ellipsis), abi) -def _func_key_hash(unique_cache, fargs, fresult, ellipsis): +def _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi): x = compute_identity_hash(fresult) for w_arg in fargs: y = compute_identity_hash(w_arg) x = intmask((1000003 * x) ^ y) - x ^= ellipsis + x ^= (ellipsis - abi) if unique_cache.for_testing: # constant-folded to False in translation; x &= 3 # but for test, keep only 2 bits of hash return x # can't use @jit.elidable here, because it might call back to random # space functions via force_lazy_struct() -def _new_function_type(space, fargs, fresult, ellipsis=False): +def _new_function_type(space, fargs, fresult, ellipsis, abi): try: - return _get_function_type(space, fargs, fresult, ellipsis) + return _get_function_type(space, fargs, fresult, ellipsis, abi) except KeyError: - return _build_function_type(space, fargs, fresult, ellipsis) + return _build_function_type(space, fargs, fresult, ellipsis, abi) @jit.elidable -def _get_function_type(space, fargs, fresult, ellipsis): +def _get_function_type(space, fargs, fresult, ellipsis, abi): # This function is elidable because if called again with exactly the # same arguments (and if it didn't raise KeyError), it would give # the same result, at least as long as this result is still live. @@ -633,18 +635,19 @@ # one such dict, but in case of hash collision, there might be # more. unique_cache = space.fromcache(UniqueCache) - func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis) + func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi) for weakdict in unique_cache.functions: ctype = weakdict.get(func_hash) if (ctype is not None and ctype.ctitem is fresult and ctype.fargs == fargs and - ctype.ellipsis == ellipsis): + ctype.ellipsis == ellipsis and + ctype.abi == abi): return ctype raise KeyError @jit.dont_look_inside -def _build_function_type(space, fargs, fresult, ellipsis): +def _build_function_type(space, fargs, fresult, ellipsis, abi): from pypy.module._cffi_backend import ctypefunc # if ((fresult.size < 0 and @@ -658,9 +661,9 @@ raise oefmt(space.w_TypeError, "invalid result type: '%s'", fresult.name) # - fct = ctypefunc.W_CTypeFunc(space, fargs, fresult, ellipsis) + fct = ctypefunc.W_CTypeFunc(space, fargs, fresult, ellipsis, abi) unique_cache = space.fromcache(UniqueCache) - func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis) + func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi) for weakdict in unique_cache.functions: if weakdict.get(func_hash) is None: weakdict.set(func_hash, fct) diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py --- a/pypy/module/_cffi_backend/realize_c_type.py +++ b/pypy/module/_cffi_backend/realize_c_type.py @@ -5,6 +5,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import oefmt from pypy.interpreter.baseobjspace import W_Root +from pypy.module import _cffi_backend from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend import cffi_opcode, newtype, ctypestruct from pypy.module._cffi_backend import parse_c_type @@ -164,16 +165,28 @@ OP_FUNCTION_END = cffi_opcode.OP_FUNCTION_END while getop(opcodes[base_index + num_args]) != OP_FUNCTION_END: num_args += 1 - ellipsis = (getarg(opcodes[base_index + num_args]) & 1) != 0 + # + ellipsis = (getarg(opcodes[base_index + num_args]) & 0x01) != 0 + abi = (getarg(opcodes[base_index + num_args]) & 0xFE) + if abi == 0: + abi = _cffi_backend.FFI_DEFAULT_ABI + elif abi == 2: + if _cffi_backend.has_stdcall: + abi = _cffi_backend.FFI_STDCALL + else: + abi = _cffi_backend.FFI_DEFAULT_ABI + else: + raise oefmt(ffi.w_FFIError, "abi number %d not supported", abi) + # fargs = [realize_c_type(ffi, opcodes, base_index + i) for i in range(num_args)] - return fargs, fret, ellipsis + return fargs, fret, ellipsis, abi def unwrap_as_fnptr(self, ffi): if self._ctfuncptr is None: - fargs, fret, ellipsis = self._unpack(ffi) + fargs, fret, ellipsis, abi = self._unpack(ffi) self._ctfuncptr = newtype._new_function_type( - ffi.space, fargs, fret, ellipsis) + ffi.space, fargs, fret, ellipsis, abi) return self._ctfuncptr def unwrap_as_fnptr_in_elidable(self): @@ -190,7 +203,7 @@ # type ptr-to-struct. This is how recompiler.py produces # trampoline functions for PyPy. if self.nostruct_ctype is None: - fargs, fret, ellipsis = self._unpack(ffi) + fargs, fret, ellipsis, abi = self._unpack(ffi) # 'locs' will be a string of the same length as the final fargs, # containing 'A' where a struct argument was detected, and 'R' # in first position if a struct return value was detected @@ -207,7 +220,7 @@ locs = ['R'] + locs fret = newtype.new_void_type(ffi.space) ctfuncptr = newtype._new_function_type( - ffi.space, fargs, fret, ellipsis) + ffi.space, fargs, fret, ellipsis, abi) if locs == ['\x00'] * len(locs): locs = None else: @@ -218,7 +231,7 @@ locs[0] == 'R') def unexpected_fn_type(self, ffi): - fargs, fret, ellipsis = self._unpack(ffi) + fargs, fret, ellipsis, abi = self._unpack(ffi) argnames = [farg.name for farg in fargs] if ellipsis: argnames.append('...') diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c --- a/pypy/module/_cffi_backend/src/parse_c_type.c +++ b/pypy/module/_cffi_backend/src/parse_c_type.c @@ -51,6 +51,9 @@ TOK_UNSIGNED, TOK_VOID, TOK_VOLATILE, + + TOK_CDECL, + TOK_STDCALL, }; typedef struct { @@ -165,6 +168,8 @@ switch (*p) { case '_': if (tok->size == 5 && !memcmp(p, "_Bool", 5)) tok->kind = TOK__BOOL; + if (tok->size == 7 && !memcmp(p,"__cdecl",7)) tok->kind = TOK_CDECL; + if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL; break; case 'c': if (tok->size == 4 && !memcmp(p, "char", 4)) tok->kind = TOK_CHAR; @@ -236,7 +241,7 @@ type). The 'outer' argument is the index of the opcode outside this "sequel". */ - int check_for_grouping; + int check_for_grouping, abi=0; _cffi_opcode_t result, *p_current; header: @@ -253,6 +258,12 @@ /* ignored for now */ next_token(tok); goto header; + case TOK_CDECL: + case TOK_STDCALL: + /* must be in a function; checked below */ + abi = tok->kind; + next_token(tok); + goto header; default: break; } @@ -269,6 +280,11 @@ while (tok->kind == TOK_OPEN_PAREN) { next_token(tok); + if (tok->kind == TOK_CDECL || tok->kind == TOK_STDCALL) { + abi = tok->kind; + next_token(tok); + } + if ((check_for_grouping--) == 1 && (tok->kind == TOK_STAR || tok->kind == TOK_CONST || tok->kind == TOK_VOLATILE || @@ -286,7 +302,14 @@ } else { /* function type */ - int arg_total, base_index, arg_next, has_ellipsis=0; + int arg_total, base_index, arg_next, flags=0; + + if (abi == TOK_STDCALL) { + flags = 2; + /* note that an ellipsis below will overwrite this flags, + which is the goal: variadic functions are always cdecl */ + } + abi = 0; if (tok->kind == TOK_VOID && get_following_char(tok) == ')') { next_token(tok); @@ -315,7 +338,7 @@ _cffi_opcode_t oarg; if (tok->kind == TOK_DOTDOTDOT) { - has_ellipsis = 1; + flags = 1; /* ellipsis */ next_token(tok); break; } @@ -339,8 +362,7 @@ next_token(tok); } } - tok->output[arg_next] = _CFFI_OP(_CFFI_OP_FUNCTION_END, - has_ellipsis); + tok->output[arg_next] = _CFFI_OP(_CFFI_OP_FUNCTION_END, flags); } if (tok->kind != TOK_CLOSE_PAREN) @@ -348,6 +370,9 @@ next_token(tok); } + if (abi != 0) + return parse_error(tok, "expected '('"); + while (tok->kind == TOK_OPEN_BRACKET) { *p_current = _CFFI_OP(_CFFI_GETOP(*p_current), tok->output_index); p_current = tok->output + tok->output_index; diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -2316,9 +2316,6 @@ f(); f() assert get_errno() == 77 -def test_abi(): - assert isinstance(FFI_DEFAULT_ABI, int) - def test_cast_to_array(): # not valid in C! extension to get a non-owning <cdata 'int[3]'> BInt = new_primitive_type("int") @@ -3396,6 +3393,78 @@ check(4 | 8, "CHB", "GTB") check(4 | 16, "CHB", "ROB") +def test_memmove(): + Short = new_primitive_type("short") + ShortA = new_array_type(new_pointer_type(Short), None) + Char = new_primitive_type("char") + CharA = new_array_type(new_pointer_type(Char), None) + p = newp(ShortA, [-1234, -2345, -3456, -4567, -5678]) + memmove(p, p + 1, 4) + assert list(p) == [-2345, -3456, -3456, -4567, -5678] + p[2] = 999 + memmove(p + 2, p, 6) + assert list(p) == [-2345, -3456, -2345, -3456, 999] + memmove(p + 4, newp(CharA, b"\x71\x72"), 2) + if sys.byteorder == 'little': + assert list(p) == [-2345, -3456, -2345, -3456, 0x7271] + else: + assert list(p) == [-2345, -3456, -2345, -3456, 0x7172] + +def test_memmove_buffer(): + import array + Short = new_primitive_type("short") + ShortA = new_array_type(new_pointer_type(Short), None) + a = array.array('H', [10000, 20000, 30000]) + p = newp(ShortA, 5) + memmove(p, a, 6) + assert list(p) == [10000, 20000, 30000, 0, 0] + memmove(p + 1, a, 6) + assert list(p) == [10000, 10000, 20000, 30000, 0] + b = array.array('h', [-1000, -2000, -3000]) + memmove(b, a, 4) + assert b.tolist() == [10000, 20000, -3000] + assert a.tolist() == [10000, 20000, 30000] + p[0] = 999 + p[1] = 998 + p[2] = 997 + p[3] = 996 + p[4] = 995 + memmove(b, p, 2) + assert b.tolist() == [999, 20000, -3000] + memmove(b, p + 2, 4) + assert b.tolist() == [997, 996, -3000] + p[2] = -p[2] + p[3] = -p[3] + memmove(b, p + 2, 6) + assert b.tolist() == [-997, -996, 995] + +def test_memmove_readonly_readwrite(): + SignedChar = new_primitive_type("signed char") + SignedCharA = new_array_type(new_pointer_type(SignedChar), None) + p = newp(SignedCharA, 5) + memmove(p, b"abcde", 3) + assert list(p) == [ord("a"), ord("b"), ord("c"), 0, 0] + memmove(p, bytearray(b"ABCDE"), 2) + assert list(p) == [ord("A"), ord("B"), ord("c"), 0, 0] + py.test.raises((TypeError, BufferError), memmove, b"abcde", p, 3) + ba = bytearray(b"xxxxx") + memmove(dest=ba, src=p, n=3) + assert ba == bytearray(b"ABcxx") + memmove(ba, b"EFGH", 4) + assert ba == bytearray(b"EFGHx") + +def test_memmove_sign_check(): + SignedChar = new_primitive_type("signed char") + SignedCharA = new_array_type(new_pointer_type(SignedChar), None) + p = newp(SignedCharA, 5) + py.test.raises(ValueError, memmove, p, p + 1, -1) # not segfault + +def test_memmove_bad_cdata(): + BInt = new_primitive_type("int") + p = cast(BInt, 42) + py.test.raises(TypeError, memmove, p, bytearray(b'a'), 1) + py.test.raises(TypeError, memmove, bytearray(b'a'), p, 1) + def test_dereference_null_ptr(): BInt = new_primitive_type("int") BIntPtr = new_pointer_type(BInt) @@ -3427,3 +3496,16 @@ "be 'foo *', but the types are different (check " "that you are not e.g. mixing up different ffi " "instances)") + +def test_stdcall_function_type(): + assert FFI_CDECL == FFI_DEFAULT_ABI + try: + stdcall = FFI_STDCALL + except NameError: + stdcall = FFI_DEFAULT_ABI + BInt = new_primitive_type("int") + BFunc = new_function_type((BInt, BInt), BInt, False, stdcall) + if stdcall != FFI_DEFAULT_ABI: + assert repr(BFunc) == "<ctype 'int(__stdcall *)(int, int)'>" + else: + assert repr(BFunc) == "<ctype 'int(*)(int, int)'>" diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -247,6 +247,63 @@ ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + def test_memmove(self): + import sys + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + p = ffi.new("short[]", [-1234, -2345, -3456, -4567, -5678]) + ffi.memmove(p, p + 1, 4) + assert list(p) == [-2345, -3456, -3456, -4567, -5678] + p[2] = 999 + ffi.memmove(p + 2, p, 6) + assert list(p) == [-2345, -3456, -2345, -3456, 999] + ffi.memmove(p + 4, ffi.new("char[]", b"\x71\x72"), 2) + if sys.byteorder == 'little': + assert list(p) == [-2345, -3456, -2345, -3456, 0x7271] + else: + assert list(p) == [-2345, -3456, -2345, -3456, 0x7172] + + def test_memmove_buffer(self): + import _cffi_backend as _cffi1_backend + import array + ffi = _cffi1_backend.FFI() + a = array.array('H', [10000, 20000, 30000]) + p = ffi.new("short[]", 5) + ffi.memmove(p, a, 6) + assert list(p) == [10000, 20000, 30000, 0, 0] + ffi.memmove(p + 1, a, 6) + assert list(p) == [10000, 10000, 20000, 30000, 0] + b = array.array('h', [-1000, -2000, -3000]) + ffi.memmove(b, a, 4) + assert b.tolist() == [10000, 20000, -3000] + assert a.tolist() == [10000, 20000, 30000] + p[0] = 999 + p[1] = 998 + p[2] = 997 + p[3] = 996 + p[4] = 995 + ffi.memmove(b, p, 2) + assert b.tolist() == [999, 20000, -3000] + ffi.memmove(b, p + 2, 4) + assert b.tolist() == [997, 996, -3000] + p[2] = -p[2] + p[3] = -p[3] + ffi.memmove(b, p + 2, 6) + assert b.tolist() == [-997, -996, 995] + + def test_memmove_readonly_readwrite(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + p = ffi.new("signed char[]", 5) + ffi.memmove(p, b"abcde", 3) + assert list(p) == [ord("a"), ord("b"), ord("c"), 0, 0] + ffi.memmove(p, bytearray(b"ABCDE"), 2) + assert list(p) == [ord("A"), ord("B"), ord("c"), 0, 0] + raises((TypeError, BufferError), ffi.memmove, b"abcde", p, 3) + ba = bytearray(b"xxxxx") + ffi.memmove(dest=ba, src=p, n=3) + assert ba == bytearray(b"ABcxx") + def test_ffi_types(self): import _cffi_backend as _cffi1_backend CData = _cffi1_backend.FFI.CData diff --git a/pypy/module/_cffi_backend/test/test_handle.py b/pypy/module/_cffi_backend/test/test_handle.py deleted file mode 100644 --- a/pypy/module/_cffi_backend/test/test_handle.py +++ /dev/null @@ -1,44 +0,0 @@ -import random -from pypy.module._cffi_backend.handle import CffiHandles - - -class PseudoWeakRef(object): - _content = 42 - - def __call__(self): - return self._content - - -def test_cffi_handles_1(): - ch = CffiHandles(None) - expected_content = {} - for i in range(10000): - index = ch.reserve_next_handle_index() - assert 0 <= index < len(ch.handles) - assert ch.handles[index]() is None - pwr = PseudoWeakRef() - expected_content[index] = pwr - ch.handles[index] = pwr - assert len(ch.handles) <= 16384 - for index, pwr in expected_content.items(): - assert ch.handles[index] is pwr - -def test_cffi_handles_2(): - ch = CffiHandles(None) - expected_content = {} - for i in range(10000): - index = ch.reserve_next_handle_index() - assert 0 <= index < len(ch.handles) - assert ch.handles[index]() is None - pwr = PseudoWeakRef() - expected_content[index] = pwr - ch.handles[index] = pwr - # - if len(expected_content) > 20: - r = random.choice(list(expected_content)) - pwr = expected_content.pop(r) - pwr._content = None - # - assert len(ch.handles) < 100 - for index, pwr in expected_content.items(): - assert ch.handles[index] is pwr diff --git a/pypy/module/_cffi_backend/test/test_parse_c_type.py b/pypy/module/_cffi_backend/test/test_parse_c_type.py --- a/pypy/module/_cffi_backend/test/test_parse_c_type.py +++ b/pypy/module/_cffi_backend/test/test_parse_c_type.py @@ -338,3 +338,17 @@ # not supported (really obscure): # "char[+5]" # "char['A']" + +def test_stdcall_cdecl(): + assert parse("int __stdcall(int)") == [Prim(cffi_opcode.PRIM_INT), + '->', Func(0), NoOp(4), FuncEnd(2), + Prim(cffi_opcode.PRIM_INT)] + assert parse("int __stdcall func(int)") == parse("int __stdcall(int)") + assert parse("int (__stdcall *)()") == [Prim(cffi_opcode.PRIM_INT), + NoOp(3), '->', Pointer(1), + Func(0), FuncEnd(2), 0] + assert parse("int (__stdcall *p)()") == parse("int (__stdcall*)()") + parse_error("__stdcall int", "identifier expected", 0) + parse_error("__cdecl int", "identifier expected", 0) + parse_error("int __stdcall", "expected '('", 13) + parse_error("int __cdecl", "expected '('", 11) diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -118,7 +118,7 @@ if buffering < 0: buffering = DEFAULT_BUFFER_SIZE - if space.config.translation.type_system == 'lltype' and 'st_blksize' in STAT_FIELD_TYPES: + if 'st_blksize' in STAT_FIELD_TYPES: fileno = space.c_int_w(space.call_method(w_raw, "fileno")) try: st = os.fstat(fileno) diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py --- a/pypy/module/cpyext/ndarrayobject.py +++ b/pypy/module/cpyext/ndarrayobject.py @@ -12,6 +12,7 @@ from pypy.module.micronumpy.descriptor import get_dtype_cache, W_Dtype from pypy.module.micronumpy.concrete import ConcreteArray from pypy.module.micronumpy import ufuncs +import pypy.module.micronumpy.constants as NPY from rpython.rlib.rawstorage import RAW_STORAGE_PTR from pypy.interpreter.typedef import TypeDef from pypy.interpreter.baseobjspace import W_Root @@ -203,12 +204,12 @@ return shape, dtype def simple_new(space, nd, dims, typenum, - order='C', owning=False, w_subtype=None): + order=NPY.CORDER, owning=False, w_subtype=None): shape, dtype = get_shape_and_dtype(space, nd, dims, typenum) return W_NDimArray.from_shape(space, shape, dtype) def simple_new_from_data(space, nd, dims, typenum, data, - order='C', owning=False, w_subtype=None): + order=NPY.CORDER, owning=False, w_subtype=None): shape, dtype = get_shape_and_dtype(space, nd, dims, typenum) storage = rffi.cast(RAW_STORAGE_PTR, data) return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, @@ -238,7 +239,7 @@ raise OperationError(space.w_NotImplementedError, space.wrap("strides must be NULL")) - order = 'C' if flags & NPY_C_CONTIGUOUS else 'F' + order = NPY.CORDER if flags & NPY_C_CONTIGUOUS else NPY.FORTRANORDER owning = True if flags & NPY_OWNDATA else False w_subtype = None diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py --- a/pypy/module/cpyext/test/test_ndarrayobject.py +++ b/pypy/module/cpyext/test/test_ndarrayobject.py @@ -4,16 +4,17 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.micronumpy.ndarray import W_NDimArray from pypy.module.micronumpy.descriptor import get_dtype_cache +import pypy.module.micronumpy.constants as NPY def scalar(space): dtype = get_dtype_cache(space).w_float64dtype return W_NDimArray.new_scalar(space, dtype, space.wrap(10.)) -def array(space, shape, order='C'): +def array(space, shape, order=NPY.CORDER): dtype = get_dtype_cache(space).w_float64dtype return W_NDimArray.from_shape(space, shape, dtype, order=order) -def iarray(space, shape, order='C'): +def iarray(space, shape, order=NPY.CORDER): dtype = get_dtype_cache(space).w_int64dtype return W_NDimArray.from_shape(space, shape, dtype, order=order) @@ -32,8 +33,8 @@ def test_FLAGS(self, space, api): s = array(space, [10]) - c = array(space, [10, 5, 3], order='C') - f = array(space, [10, 5, 3], order='F') + c = array(space, [10, 5, 3], order=NPY.CORDER) + f = array(space, [10, 5, 3], order=NPY.FORTRANORDER) assert api._PyArray_FLAGS(s) & 0x0001 assert api._PyArray_FLAGS(s) & 0x0002 assert api._PyArray_FLAGS(c) & 0x0001 diff --git a/pypy/module/itertools/__init__.py b/pypy/module/itertools/__init__.py --- a/pypy/module/itertools/__init__.py +++ b/pypy/module/itertools/__init__.py @@ -10,7 +10,6 @@ repeat(elem [,n]) --> elem, elem, elem, ... endlessly or up to n times Iterators terminating on the shortest input sequence: - izip(p, q, ...) --> (p[0], q[0]), (p[1], q[1]), ... ifilter(pred, seq) --> elements of seq where pred(elem) is True ifilterfalse(pred, seq) --> elements of seq where pred(elem) is False islice(seq, [start,] stop [, step]) --> elements from @@ -22,6 +21,14 @@ takewhile(pred, seq) --> seq[0], seq[1], until pred fails dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v) + izip(p, q, ...) --> (p[0], q[0]), (p[1], q[1]), ... + izip_longest(p, q, ...) --> (p[0], q[0]), (p[1], q[1]), ... + + Combinatoric generators: + product(p, q, ... [repeat=1]) --> cartesian product + permutations(p[, r]) + combinations(p, r) + combinations_with_replacement(p, r) """ interpleveldefs = { diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py --- a/pypy/module/itertools/interp_itertools.py +++ b/pypy/module/itertools/interp_itertools.py @@ -649,33 +649,38 @@ class W_IZipLongest(W_IMap): _error_name = "izip_longest" + _immutable_fields_ = ["w_fillvalue"] + + def _fetch(self, index): + w_iter = self.iterators_w[index] + if w_iter is not None: + space = self.space + try: + return space.next(w_iter) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + self.active -= 1 + if self.active <= 0: + # It was the last active iterator + raise + self.iterators_w[index] = None + return self.w_fillvalue def next_w(self): - space = self.space + # common case: 2 arguments + if len(self.iterators_w) == 2: + objects = [self._fetch(0), self._fetch(1)] + else: + objects = self._get_objects() + return self.space.newtuple(objects) + + def _get_objects(self): + # the loop is out of the way of the JIT nb = len(self.iterators_w) - if nb == 0: - raise OperationError(space.w_StopIteration, space.w_None) - - objects_w = [None] * nb - for index in range(nb): - w_value = self.w_fillvalue - w_it = self.iterators_w[index] - if w_it is not None: - try: - w_value = space.next(w_it) - except OperationError, e: - if not e.match(space, space.w_StopIteration): - raise - - self.active -= 1 - if self.active == 0: - # It was the last active iterator - raise - self.iterators_w[index] = None - - objects_w[index] = w_value - return space.newtuple(objects_w) + raise OperationError(self.space.w_StopIteration, self.space.w_None) + return [self._fetch(index) for index in range(nb)] def W_IZipLongest___new__(space, w_subtype, __args__): arguments_w, kwds_w = __args__.unpack() diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -108,7 +108,8 @@ w_axis = space.wrap(0) if space.is_none(w_axis): args_w = [w_arg.reshape(space, - space.newlist([w_arg.descr_get_size(space)])) + space.newlist([w_arg.descr_get_size(space)]), + w_arg.get_order()) for w_arg in args_w] w_axis = space.wrap(0) dtype = args_w[0].get_dtype() @@ -140,7 +141,7 @@ dtype = find_result_type(space, args_w, []) # concatenate does not handle ndarray subtypes, it always returns a ndarray - res = W_NDimArray.from_shape(space, shape, dtype, 'C') + res = W_NDimArray.from_shape(space, shape, dtype, NPY.CORDER) chunks = [Chunk(0, i, 1, i) for i in shape] axis_start = 0 for arr in args_w: diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -38,7 +38,8 @@ self.implementation = implementation @staticmethod - def from_shape(space, shape, dtype, order='C', w_instance=None, zero=True): + def from_shape(space, shape, dtype, order=NPY.CORDER, + w_instance=None, zero=True): from pypy.module.micronumpy import concrete, descriptor, boxes from pypy.module.micronumpy.strides import calc_strides if len(shape) > NPY.MAXDIMS: @@ -59,8 +60,9 @@ @staticmethod def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1, - order='C', owning=False, w_subtype=None, - w_base=None, writable=True, strides=None, start=0): + order=NPY.CORDER, owning=False, w_subtype=None, + w_base=None, writable=True, strides=None, + start=0): from pypy.module.micronumpy import concrete from pypy.module.micronumpy.strides import (calc_strides, calc_backstrides) diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -56,6 +56,9 @@ jit.hint(len(backstrides), promote=True) return backstrides + def get_flags(self): + return self.flags + def getitem(self, index): return self.dtype.read(self, index, 0) @@ -89,17 +92,18 @@ def get_storage_size(self): return self.size - def reshape(self, orig_array, new_shape): + def reshape(self, orig_array, new_shape, order=NPY.ANYORDER): # Since we got to here, prod(new_shape) == self.size + order = support.get_order_as_CF(self.order, order) new_strides = None if self.size == 0: - new_strides, _ = calc_strides(new_shape, self.dtype, self.order) + new_strides, _ = calc_strides(new_shape, self.dtype, order) else: if len(self.get_shape()) == 0: new_strides = [self.dtype.elsize] * len(new_shape) else: new_strides = calc_new_strides(new_shape, self.get_shape(), - self.get_strides(), self.order) + self.get_strides(), order) if new_strides is None or len(new_strides) != len(new_shape): return None if new_strides is not None: @@ -303,10 +307,11 @@ return SliceArray(self.start, strides, backstrides, shape, self, orig_array) - def copy(self, space): + def copy(self, space, order=NPY.ANYORDER): + order = support.get_order_as_CF(self.order, order) strides, backstrides = calc_strides(self.get_shape(), self.dtype, - self.order) - impl = ConcreteArray(self.get_shape(), self.dtype, self.order, strides, + order) + impl = ConcreteArray(self.get_shape(), self.dtype, order, strides, backstrides) return loop.setslice(space, self.get_shape(), impl, self) @@ -360,12 +365,12 @@ # but make the array storage contiguous in memory shape = self.get_shape() strides = self.get_strides() - if order not in ('C', 'F'): - raise oefmt(space.w_ValueError, "Unknown order %s in astype", order) + if order not in (NPY.KEEPORDER, NPY.FORTRANORDER, NPY.CORDER): + raise oefmt(space.w_ValueError, "Unknown order %d in astype", order) if len(strides) == 0: t_strides = [] backstrides = [] - elif order != self.order: + elif order in (NPY.FORTRANORDER, NPY.CORDER): t_strides, backstrides = calc_strides(shape, dtype, order) else: indx_array = range(len(strides)) @@ -378,6 +383,7 @@ t_strides[i] = base base *= shape[i] backstrides = calc_backstrides(t_strides, shape) + order = support.get_order_as_CF(self.order, order) impl = ConcreteArray(shape, dtype, order, t_strides, backstrides) loop.setslice(space, impl.get_shape(), impl, self) return impl @@ -429,6 +435,8 @@ self.shape = shape # already tested for overflow in from_shape_and_storage self.size = support.product(shape) * dtype.elsize + if order not in (NPY.CORDER, NPY.FORTRANORDER): + raise oefmt(dtype.itemtype.space.w_ValueError, "ConcreteArrayNotOwning but order is not 0,1 rather %d", order) self.order = order self.dtype = dtype self.strides = strides @@ -562,6 +570,8 @@ self.parent = parent self.storage = parent.storage self.gcstruct = parent.gcstruct + if parent.order not in (NPY.CORDER, NPY.FORTRANORDER): + raise oefmt(dtype.itemtype.space.w_ValueError, "SliceArray but parent order is not 0,1 rather %d", parent.order) self.order = parent.order self.dtype = dtype try: @@ -602,13 +612,13 @@ s = self.get_strides()[0] // dtype.elsize except IndexError: s = 1 - if self.order == 'C': + if self.order != NPY.FORTRANORDER: new_shape.reverse() for sh in new_shape: strides.append(s * dtype.elsize) backstrides.append(s * (sh - 1) * dtype.elsize) s *= max(1, sh) - if self.order == 'C': + if self.order != NPY.FORTRANORDER: strides.reverse() backstrides.reverse() new_shape.reverse() diff --git a/pypy/module/micronumpy/converters.py b/pypy/module/micronumpy/converters.py --- a/pypy/module/micronumpy/converters.py +++ b/pypy/module/micronumpy/converters.py @@ -77,9 +77,8 @@ elif order.startswith('K') or order.startswith('k'): return NPY.KEEPORDER else: - raise OperationError(space.w_TypeError, space.wrap( - "order not understood")) - + raise oefmt(space.w_TypeError, "Unknown order: '%s'", order) + return -1 def multi_axis_converter(space, w_axis, ndim): if space.is_none(w_axis): diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -3,10 +3,13 @@ from rpython.rlib.buffer import SubBuffer from rpython.rlib.rstring import strip_spaces from rpython.rtyper.lltypesystem import lltype, rffi + from pypy.module.micronumpy import descriptor, loop, support -from pypy.module.micronumpy.base import ( +from pypy.module.micronumpy.base import (wrap_impl, W_NDimArray, convert_to_array, W_NumpyObject) -from pypy.module.micronumpy.converters import shape_converter +from pypy.module.micronumpy.converters import shape_converter, order_converter +import pypy.module.micronumpy.constants as NPY +from .casting import scalar2dtype def build_scalar(space, w_dtype, w_state): @@ -82,7 +85,6 @@ return w_res def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False): - from pypy.module.micronumpy import strides # for anything that isn't already an array, try __array__ method first if not isinstance(w_object, W_NDimArray): @@ -99,13 +101,8 @@ dtype = descriptor.decode_w_dtype(space, w_dtype) if space.is_none(w_order): - order = 'C' - else: - order = space.str_w(w_order) - if order == 'K': - order = 'C' - if order != 'C': # or order != 'F': - raise oefmt(space.w_ValueError, "Unknown order: %s", order) + w_order = space.wrap('C') + npy_order = order_converter(space, w_order, NPY.CORDER) if isinstance(w_object, W_NDimArray): if (dtype is None or w_object.get_dtype() is dtype): @@ -124,7 +121,7 @@ copy = True if copy: shape = w_object.get_shape() - w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order) + w_arr = W_NDimArray.from_shape(space, shape, dtype, order=npy_order) if support.product(shape) == 1: w_arr.set_scalar_value(dtype.coerce(space, w_object.implementation.getitem(0))) @@ -143,18 +140,13 @@ w_base=w_base, start=imp.start) else: # not an array - shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) + shape, elems_w = find_shape_and_elems(space, w_object, dtype) if dtype is None and space.isinstance_w(w_object, space.w_buffer): dtype = descriptor.get_dtype_cache(space).w_uint8dtype if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1): dtype = find_dtype_for_seq(space, elems_w, dtype) - if dtype is None: - dtype = descriptor.get_dtype_cache(space).w_float64dtype - elif dtype.is_str_or_unicode() and dtype.elsize < 1: - # promote S0 -> S1, U0 -> U1 - dtype = descriptor.variable_dtype(space, dtype.char + '1') - w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order) + w_arr = W_NDimArray.from_shape(space, shape, dtype, order=npy_order) if support.product(shape) == 1: # safe from overflow since from_shape checks w_arr.set_scalar_value(dtype.coerce(space, elems_w[0])) else: @@ -165,7 +157,6 @@ def numpify(space, w_object): """Convert the object to a W_NumpyObject""" # XXX: code duplication with _array() - from pypy.module.micronumpy import strides if isinstance(w_object, W_NumpyObject): return w_object # for anything that isn't already an array, try __array__ method first @@ -173,20 +164,82 @@ if w_array is not None: return w_array - shape, elems_w = strides.find_shape_and_elems(space, w_object, None) + if is_scalar_like(space, w_object, dtype=None): + dtype = scalar2dtype(space, w_object) + if dtype.is_str_or_unicode() and dtype.elsize < 1: + # promote S0 -> S1, U0 -> U1 + dtype = descriptor.variable_dtype(space, dtype.char + '1') + return dtype.coerce(space, w_object) + + shape, elems_w = _find_shape_and_elems(space, w_object) dtype = find_dtype_for_seq(space, elems_w, None) - if dtype is None: - dtype = descriptor.get_dtype_cache(space).w_float64dtype - elif dtype.is_str_or_unicode() and dtype.elsize < 1: - # promote S0 -> S1, U0 -> U1 - dtype = descriptor.variable_dtype(space, dtype.char + '1') + w_arr = W_NDimArray.from_shape(space, shape, dtype) + loop.assign(space, w_arr, elems_w) + return w_arr - if len(elems_w) == 1: - return dtype.coerce(space, elems_w[0]) + +def find_shape_and_elems(space, w_iterable, dtype): + if is_scalar_like(space, w_iterable, dtype): + return [], [w_iterable] + is_rec_type = dtype is not None and dtype.is_record() + return _find_shape_and_elems(space, w_iterable, is_rec_type) + +def is_scalar_like(space, w_obj, dtype): + isstr = space.isinstance_w(w_obj, space.w_str) + if not support.issequence_w(space, w_obj) or isstr: + if dtype is None or dtype.char != NPY.CHARLTR: + return True + is_rec_type = dtype is not None and dtype.is_record() + if is_rec_type and is_single_elem(space, w_obj, is_rec_type): + return True + if isinstance(w_obj, W_NDimArray) and w_obj.is_scalar(): + return True + return False + +def _find_shape_and_elems(space, w_iterable, is_rec_type=False): + from pypy.objspace.std.bufferobject import W_Buffer + shape = [space.len_w(w_iterable)] + if space.isinstance_w(w_iterable, space.w_buffer): + batch = [space.wrap(0)] * shape[0] + for i in range(shape[0]): + batch[i] = space.ord(space.getitem(w_iterable, space.wrap(i))) else: - w_arr = W_NDimArray.from_shape(space, shape, dtype) - loop.assign(space, w_arr, elems_w) - return w_arr + batch = space.listview(w_iterable) + while True: + if not batch: + return shape[:], [] + if is_single_elem(space, batch[0], is_rec_type): + for w_elem in batch: + if not is_single_elem(space, w_elem, is_rec_type): + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + return shape[:], batch + new_batch = [] + size = space.len_w(batch[0]) + for w_elem in batch: + if (is_single_elem(space, w_elem, is_rec_type) or + space.len_w(w_elem) != size): + raise OperationError(space.w_ValueError, space.wrap( + "setting an array element with a sequence")) + w_array = space.lookup(w_elem, '__array__') + if w_array is not None: + # Make sure we call the array implementation of listview, + # since for some ndarray subclasses (matrix, for instance) + # listview does not reduce but rather returns the same class + w_elem = space.get_and_call_function(w_array, w_elem, space.w_None) + new_batch += space.listview(w_elem) + shape.append(size) + batch = new_batch + +def is_single_elem(space, w_elem, is_rec_type): + if (is_rec_type and space.isinstance_w(w_elem, space.w_tuple)): + return True + if (space.isinstance_w(w_elem, space.w_tuple) or + space.isinstance_w(w_elem, space.w_list)): + return False + if isinstance(w_elem, W_NDimArray) and not w_elem.is_scalar(): _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit