Author: Armin Rigo <ar...@tunes.org> Branch: Changeset: r67912:d94f6f211aa5 Date: 2013-11-09 23:04 +0100 http://bitbucket.org/pypy/pypy/changeset/d94f6f211aa5/
Log: Import cffi/d852277c4508 diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.7.2" -__version_info__ = (0, 7, 2) +__version__ = "0.8" +__version_info__ = (0, 8) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,5 @@ import types +from .lock import allocate_lock try: callable @@ -61,6 +62,7 @@ # rely on it! It's probably not going to work well.) self._backend = backend + self._lock = allocate_lock() self._parser = cparser.Parser() self._cached_btypes = {} self._parsed_types = types.ModuleType('parsed_types').__dict__ @@ -74,7 +76,8 @@ if name.startswith('RTLD_'): setattr(self, name, getattr(backend, name)) # - self.BVoidP = self._get_cached_btype(model.voidp_type) + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -95,11 +98,12 @@ if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') - self._parser.parse(csource, override=override) - self._cdefsources.append(csource) - if override: - for cache in self._function_caches: - cache.clear() + with self._lock: + self._parser.parse(csource, override=override) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() def dlopen(self, name, flags=0): """Load and return a dynamic library identified by 'name'. @@ -109,31 +113,49 @@ library we only look for the actual (untyped) symbols. """ assert isinstance(name, basestring) or name is None - lib, function_cache = _make_ffi_library(self, name, flags) - self._function_caches.append(function_cache) - self._libraries.append(lib) + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) return lib + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + if hasattr(type, 'as_function_pointer'): + really_a_function_type = True + type = type.as_function_pointer() + else: + really_a_function_type = False + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + def _typeof(self, cdecl, consider_function_as_funcptr=False): # string -> ctype object try: - btype, cfaf = self._parsed_types[cdecl] - if consider_function_as_funcptr and not cfaf: - raise KeyError + result = self._parsed_types[cdecl] except KeyError: - key = cdecl - if not isinstance(cdecl, str): # unicode, on Python 2 - cdecl = cdecl.encode('ascii') - cfaf = consider_function_as_funcptr - type = self._parser.parse_type(cdecl, - consider_function_as_funcptr=cfaf) - btype = self._get_cached_btype(type) - self._parsed_types[key] = btype, cfaf + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) return btype def typeof(self, cdecl): """Parse the C type given as a string and return the - corresponding Python type: <class 'ffi.CData<...>'>. + corresponding <ctype> object. It can also be used on 'cdata' instance to get its C type. """ if isinstance(cdecl, basestring): @@ -144,6 +166,10 @@ res = _builtin_function_type(cdecl) if res is not None: return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) raise TypeError(type(cdecl)) def sizeof(self, cdecl): @@ -280,14 +306,17 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ - try: - gc_weakrefs = self.gc_weakrefs - except AttributeError: - from .gc_weakref import GcWeakrefs - gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) - return gc_weakrefs.build(cdata, destructor) + with self._lock: + try: + gc_weakrefs = self.gc_weakrefs + except AttributeError: + from .gc_weakref import GcWeakrefs + gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self) + return gc_weakrefs.build(cdata, destructor) def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! try: BType = self._cached_btypes[type] except KeyError: @@ -322,7 +351,8 @@ def _pointer_to(self, ctype): from . import model - return model.pointer_cache(self, ctype) + with self._lock: + return model.pointer_cache(self, ctype) def addressof(self, cdata, field=None): """Return the address of a <cdata 'struct-or-union'>. @@ -342,10 +372,12 @@ variables, which must anyway be accessed directly from the lib object returned by the original FFI instance. """ - self._parser.include(ffi_to_include._parser) - self._cdefsources.append('[') - self._cdefsources.extend(ffi_to_include._cdefsources) - self._cdefsources.append(']') + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') def new_handle(self, x): return self._backend.newp_handle(self.BVoidP, x) @@ -372,7 +404,7 @@ backendlib = backend.load_library(path, flags) copied_enums = [] # - def make_accessor(name): + def make_accessor_locked(name): key = 'function ' + name if key in ffi._parser._declarations: tp = ffi._parser._declarations[key] @@ -404,11 +436,17 @@ if enumname not in library.__dict__: library.__dict__[enumname] = enumval copied_enums.append(True) + if name in library.__dict__: + return # - if name in library.__dict__: # copied from an enum value just above, - return # or multithread's race condition raise AttributeError(name) # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + make_accessor_locked(name) + # class FFILibrary(object): def __getattr__(self, name): make_accessor(name) @@ -444,4 +482,5 @@ except (KeyError, AttributeError, TypeError): return None else: - return ffi._get_cached_btype(tp) + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -142,7 +142,7 @@ if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] if line: - msg = 'cannot parse "%s"\n%s' % (line, msg) + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) @@ -217,19 +217,18 @@ # if decl.name: tp = self._get_type(node, partial_length_ok=True) - if self._is_constant_declaration(node): + if self._is_constant_globalvar(node): self._declare('constant ' + decl.name, tp) else: self._declare('variable ' + decl.name, tp) - def parse_type(self, cdecl, consider_function_as_funcptr=False): + def parse_type(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl) assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type(exprnode.type, - consider_function_as_funcptr=consider_function_as_funcptr) + return self._get_type(exprnode.type) def _declare(self, name, obj): if name in self._declarations: @@ -249,28 +248,17 @@ return model.ConstPointerType(type) return model.PointerType(type) - def _get_type(self, typenode, convert_array_to_pointer=False, - name=None, partial_length_ok=False, - consider_function_as_funcptr=False): + def _get_type(self, typenode, name=None, partial_length_ok=False): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): type = self._declarations['typedef ' + typenode.type.names[0]] - if isinstance(type, model.ArrayType): - if convert_array_to_pointer: - return type.item - else: - if (consider_function_as_funcptr and - isinstance(type, model.RawFunctionType)): - return type.as_function_pointer() return type # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type - if convert_array_to_pointer: - return self._get_type_pointer(self._get_type(typenode.type)) if typenode.dim is None: length = None else: @@ -331,10 +319,7 @@ # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type - result = self._parse_function_type(typenode, name) - if consider_function_as_funcptr: - result = result.as_function_pointer() - return result + return self._parse_function_type(typenode, name) # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): @@ -365,21 +350,24 @@ isinstance(params[0].type.type, pycparser.c_ast.IdentifierType) and list(params[0].type.type.names) == ['void']): del params[0] - args = [self._get_type(argdeclnode.type, - convert_array_to_pointer=True, - consider_function_as_funcptr=True) + args = [self._as_func_arg(self._get_type(argdeclnode.type)) for argdeclnode in params] result = self._get_type(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) - def _is_constant_declaration(self, typenode, const=False): - if isinstance(typenode, pycparser.c_ast.ArrayDecl): - return self._is_constant_declaration(typenode.type) + def _as_func_arg(self, type): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _is_constant_globalvar(self, typenode): if isinstance(typenode, pycparser.c_ast.PtrDecl): - const = 'const' in typenode.quals - return self._is_constant_declaration(typenode.type, const) + return 'const' in typenode.quals if isinstance(typenode, pycparser.c_ast.TypeDecl): - return const or 'const' in typenode.quals + return 'const' in typenode.quals return False def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): @@ -491,7 +479,7 @@ return tp def _make_partial(self, tp, nested): - if not isinstance(tp, model.StructType): + if not isinstance(tp, model.StructOrUnion): raise api.CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) @@ -511,7 +499,7 @@ if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): self._partial_length = True - return None + return '...' # raise api.FFIError("unsupported expression: expected a " "simple numeric constant") diff --git a/lib_pypy/cffi/gc_weakref.py b/lib_pypy/cffi/gc_weakref.py --- a/lib_pypy/cffi/gc_weakref.py +++ b/lib_pypy/cffi/gc_weakref.py @@ -14,6 +14,6 @@ def build(self, cdata, destructor): # make a new cdata of the same type as the original one - new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata) + new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata) self.data[ref(new_cdata, self.remove)] = destructor, cdata return new_cdata diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -192,10 +192,6 @@ _base_pattern = " const *&" _base_pattern_array = "(const *&)" - def build_backend_type(self, ffi, finishlist): - BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist) - return BPtr - const_voidp_type = ConstPointerType(void_type) @@ -216,10 +212,10 @@ self.item = item self.length = length # - if self.length is None: + if length is None or length == '...': brackets = '&[]' else: - brackets = '&[%d]' % self.length + brackets = '&[%d]' % length self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) @@ -227,6 +223,10 @@ return ArrayType(self.item, newlength) def build_backend_type(self, ffi, finishlist): + if self.length == '...': + from . import api + raise api.CDefError("cannot render the type %r: unknown length" % + (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) @@ -252,6 +252,7 @@ class StructOrUnion(StructOrUnionOrEnum): fixedlayout = None completed = False + partial = False def __init__(self, name, fldnames, fldtypes, fldbitsize): self.name = name @@ -303,20 +304,21 @@ return # not completing it: it's an opaque struct # self.completed = 1 - fldtypes = tuple(tp.get_cached_btype(ffi, finishlist) - for tp in self.fldtypes) # if self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) ffi._backend.complete_struct_or_union(BType, lst, self) # else: + fldtypes = [] fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout for i in range(len(self.fldnames)): fsize = fieldsize[i] ftype = self.fldtypes[i] # - if isinstance(ftype, ArrayType) and ftype.length is None: + if isinstance(ftype, ArrayType) and ftype.length == '...': # fix the length to match the total size BItemType = ftype.item.get_cached_btype(ffi, finishlist) nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) @@ -327,18 +329,20 @@ ftype = ftype.resolve_length(nlen) self.fldtypes = (self.fldtypes[:i] + (ftype,) + self.fldtypes[i+1:]) - BArrayType = ftype.get_cached_btype(ffi, finishlist) - fldtypes = (fldtypes[:i] + (BArrayType,) + - fldtypes[i+1:]) - continue # - bitemsize = ffi.sizeof(fldtypes[i]) - if bitemsize != fsize: - self._verification_error( - "field '%s.%s' is declared as %d bytes, but is " - "really %d bytes" % (self.name, - self.fldnames[i] or '{}', - bitemsize, fsize)) + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) ffi._backend.complete_struct_or_union(BType, lst, self, totalsize, totalalignment) @@ -348,11 +352,6 @@ from .ffiplatform import VerificationError raise VerificationError(msg) - -class StructType(StructOrUnion): - kind = 'struct' - partial = False - def check_not_partial(self): if self.partial and self.fixedlayout is None: from . import ffiplatform @@ -361,19 +360,18 @@ def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) - - return global_cache(self, ffi, 'new_struct_type', + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, self.get_official_name(), key=self) +class StructType(StructOrUnion): + kind = 'struct' + + class UnionType(StructOrUnion): kind = 'union' - def build_backend_type(self, ffi, finishlist): - finishlist.append(self) - return global_cache(self, ffi, 'new_union_type', - self.get_official_name(), key=self) - class EnumType(StructOrUnionOrEnum): kind = 'enum' @@ -387,6 +385,12 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + def check_not_partial(self): if self.partial and not self.partial_resolved: from . import ffiplatform diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -15,7 +15,7 @@ def patch_extension_kwds(self, kwds): pass - def find_module(self, module_name, path, so_suffix): + def find_module(self, module_name, path, so_suffixes): try: f, filename, descr = imp.find_module(module_name, path) except ImportError: @@ -25,7 +25,7 @@ # Note that after a setuptools installation, there are both .py # and .so files with the same basename. The code here relies on # imp.find_module() locating the .so in priority. - if descr[0] != so_suffix: + if descr[0] not in so_suffixes: return None return filename @@ -280,8 +280,8 @@ return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(tp)) elif isinstance(tp, model.ArrayType): - return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % ( - var, self._gettypenum(tp)) + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) elif isinstance(tp, model.StructType): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( @@ -464,11 +464,14 @@ prnt(' static Py_ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return _cffi_get_struct_layout(nums);') @@ -491,7 +494,7 @@ # function = getattr(module, layoutfuncname) layout = function() - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -528,9 +531,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -566,7 +570,7 @@ # constants, likely declared with '#define' def _generate_cpy_const(self, is_int, name, tp=None, category='const', - vartp=None, delayed=True): + vartp=None, delayed=True, size_too=False): prnt = self._prnt funcname = '_cffi_%s_%s' % (category, name) prnt('static int %s(PyObject *lib)' % funcname) @@ -597,6 +601,15 @@ '(unsigned long long)(%s));' % (name,)) prnt(' if (o == NULL)') prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) prnt(' Py_DECREF(o);') prnt(' if (res < 0)') @@ -677,15 +690,16 @@ def _generate_cpy_variable_collecttype(self, tp, name): if isinstance(tp, model.ArrayType): - self._do_collect_type(tp) + tp_ptr = model.PointerType(tp.item) else: tp_ptr = model.PointerType(tp) - self._do_collect_type(tp_ptr) + self._do_collect_type(tp_ptr) def _generate_cpy_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): tp_ptr = model.PointerType(tp.item) - self._generate_cpy_const(False, name, tp, vartp=tp_ptr) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = (tp.length == '...')) else: tp_ptr = model.PointerType(tp) self._generate_cpy_const(False, name, tp_ptr, category='var') @@ -694,11 +708,29 @@ _loading_cpy_variable = _loaded_noop def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the - return # sense that "a=..." is forbidden + # sense that "a=..." is forbidden + if tp.length == '...': + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a <cdata 'type *'> which we have to replace with + # a <cdata 'type[N]'> if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return # remove ptr=<cdata 'int *'> from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. - ptr = getattr(library, name) + ptr = value delattr(library, name) def getter(library): return ptr[0] diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -20,15 +20,15 @@ # up in kwds['export_symbols']. kwds.setdefault('export_symbols', self.export_symbols) - def find_module(self, module_name, path, so_suffix): - basename = module_name + so_suffix - if path is None: - path = sys.path - for dirname in path: - filename = os.path.join(dirname, basename) - if os.path.isfile(filename): - return filename - return None + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename def collect_types(self): pass # not needed in the generic engine @@ -173,6 +173,7 @@ newfunction = self._load_constant(False, tp, name, module) else: indirections = [] + base_tp = tp if any(isinstance(typ, model.StructOrUnion) for typ in tp.args): indirect_args = [] for i, typ in enumerate(tp.args): @@ -186,16 +187,18 @@ wrappername = '_cffi_f_%s' % name newfunction = module.load_function(BFunc, wrappername) for i, typ in indirections: - newfunction = self._make_struct_wrapper(newfunction, i, typ) + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) setattr(library, name, newfunction) type(library)._cffi_dir.append(name) - def _make_struct_wrapper(self, oldfunc, i, tp): + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): backend = self.ffi._backend BType = self.ffi._get_cached_btype(tp) def newfunc(*args): args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] return oldfunc(*args) + newfunc._cffi_base_type = base_tp return newfunc # ---------- @@ -252,11 +255,14 @@ prnt(' static ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, _, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) - prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) prnt(' -1') prnt(' };') prnt(' return nums[i];') @@ -270,7 +276,7 @@ return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) # - BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)") + BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0] function = module.load_function(BFunc, layoutfuncname) layout = [] num = 0 @@ -279,7 +285,7 @@ if x < 0: break layout.append(x) num += 1 - if isinstance(tp, model.StructType) and tp.partial: + if isinstance(tp, model.StructOrUnion) and tp.partial: # use the function()'s sizes and offsets to guide the # layout of the struct totalsize = layout[0] @@ -316,9 +322,10 @@ continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), "wrong offset for field %r" % (fname,)) - BField = ffi._get_cached_btype(ftype) - check(layout[i+1], ffi.sizeof(BField), - "wrong size for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) i += 2 assert i == len(layout) @@ -379,15 +386,16 @@ def _load_constant(self, is_int, tp, name, module): funcname = '_cffi_const_%s' % name if is_int: - BFunc = self.ffi.typeof("int(*)(long long*)") + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] function = module.load_function(BFunc, funcname) - p = self.ffi.new("long long*") + p = self.ffi.new(BType) negative = function(p) value = int(p[0]) if value < 0 and not negative: value += (1 << (8*self.ffi.sizeof("long long"))) else: - BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0] function = module.load_function(BFunc, funcname) value = function() return value @@ -431,10 +439,11 @@ tp.enumvalues = tuple(enumvalues) tp.partial_resolved = True else: - BFunc = self.ffi.typeof("int(*)(char*)") + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] funcname = '_cffi_e_%s_%s' % (prefix, name) function = module.load_function(BFunc, funcname) - p = self.ffi.new("char[]", 256) + p = self.ffi.new(BType, 256) if function(p) < 0: error = self.ffi.string(p) if sys.version_info >= (3,): @@ -465,6 +474,14 @@ def _generate_gen_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): + if tp.length == '...': + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") tp_ptr = model.PointerType(tp.item) self._generate_gen_const(False, name, tp_ptr) else: @@ -476,6 +493,18 @@ def _loaded_gen_variable(self, tp, name, module, library): if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the # sense that "a=..." is forbidden + if tp.length == '...': + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise ffiplatform.VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) tp_ptr = model.PointerType(tp.item) value = self._load_constant(False, tp_ptr, name, module) # 'value' is a <cdata 'type *'> which we have to replace with @@ -489,7 +518,7 @@ # remove ptr=<cdata 'int *'> from the library instance, and replace # it by a property on the class, which reads/writes into ptr[0]. funcname = '_cffi_var_%s' % name - BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name)) + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] function = module.load_function(BFunc, funcname) ptr = function() def getter(library): diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -31,7 +31,7 @@ k2 = k2.lstrip('0').rstrip('L') modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) - suffix = _get_so_suffix() + suffix = _get_so_suffixes()[0] self.tmpdir = tmpdir or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) @@ -42,18 +42,21 @@ def write_source(self, file=None): """Write the C source code. It is produced in 'self.sourcefilename', which can be tweaked beforehand.""" - if self._has_source and file is None: - raise ffiplatform.VerificationError("source code already written") - self._write_source(file) + with self.ffi._lock: + if self._has_source and file is None: + raise ffiplatform.VerificationError( + "source code already written") + self._write_source(file) def compile_module(self): """Write the C source code (if not done already) and compile it. This produces a dynamic link library in 'self.modulefilename'.""" - if self._has_module: - raise ffiplatform.VerificationError("module already compiled") - if not self._has_source: - self._write_source() - self._compile_module() + with self.ffi._lock: + if self._has_module: + raise ffiplatform.VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() def load_library(self): """Get a C module from this Verifier instance. @@ -62,11 +65,14 @@ operations to the C module. If necessary, the C code is written and compiled first. """ - if not self._has_module: - self._locate_module() + with self.ffi._lock: if not self._has_module: - self.compile_module() - return self._load_library() + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() def get_module_name(self): basename = os.path.basename(self.modulefilename) @@ -81,7 +87,9 @@ def get_extension(self): if not self._has_source: - self._write_source() + with self.ffi._lock: + if not self._has_source: + self._write_source() sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) modname = self.get_module_name() return ffiplatform.get_extension(sourcename, modname, **self.kwds) @@ -103,7 +111,7 @@ else: path = None filename = self._vengine.find_module(self.get_module_name(), path, - _get_so_suffix()) + _get_so_suffixes()) if filename is None: return self.modulefilename = filename @@ -193,7 +201,7 @@ if keep_so: suffix = '.c' # only remove .c files else: - suffix = _get_so_suffix().lower() + suffix = _get_so_suffixes()[0].lower() for fn in filelist: if fn.lower().startswith('_cffi_') and ( fn.lower().endswith(suffix) or fn.lower().endswith('.c')): @@ -213,15 +221,20 @@ except OSError: pass -def _get_so_suffix(): +def _get_so_suffixes(): + suffixes = [] for suffix, mode, type in imp.get_suffixes(): if type == imp.C_EXTENSION: - return suffix - # bah, no C_EXTENSION available. Occurs on pypy without cpyext - if sys.platform == 'win32': - return ".pyd" - else: - return ".so" + suffixes.append(suffix) + + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes def _ensure_dir(filename): try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_ffi_backend.py @@ -185,3 +185,13 @@ ffi.cdef("typedef struct { float x; } foo_t;") p = ffi.new("foo_t *", [5.2]) assert repr(p).startswith("<cdata 'foo_t *' ") + + def test_struct_array_no_length(self): + ffi = FFI() + ffi.cdef("struct foo_s { int x; int a[]; };") + p = ffi.new("struct foo_s *", [100, [200, 300, 400]]) + assert p.x == 100 + assert ffi.typeof(p.a) is ffi.typeof("int *") # no length available + assert p.a[0] == 200 + assert p.a[1] == 300 + assert p.a[2] == 400 diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/test_parsing.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_parsing.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_parsing.py @@ -131,8 +131,9 @@ ffi.cdef(""" typedef int (*fn_t)(int[5]); """) - type = ffi._parser.parse_type("fn_t") - BType = ffi._get_cached_btype(type) + with ffi._lock: + type = ffi._parser.parse_type("fn_t") + BType = ffi._get_cached_btype(type) assert str(BType) == '<func (<pointer to <int>>), <int>, False>' def test_remove_comments(): @@ -191,7 +192,7 @@ def test_parse_error(): ffi = FFI() e = py.test.raises(CDefError, ffi.cdef, " x y z ") - assert re.match(r'cannot parse " x y z "\n:\d+:', str(e.value)) + assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value)) def test_cannot_declare_enum_later(): ffi = FFI() @@ -256,3 +257,27 @@ py.test.skip("Only for Windows") ffi = FFI() ffi.cdef("void f(WPARAM);") + +def test__is_constant_globalvar(): + from cffi.cparser import Parser, _get_parser + for input, expected_output in [ + ("int a;", False), + ("const int a;", True), + ("int *a;", False), + ("const int *a;", False), + ("int const *a;", False), + ("int *const a;", True), + ("int a[5];", False), + ("const int a[5];", False), + ("int *a[5];", False), + ("const int *a[5];", False), + ("int const *a[5];", False), + ("int *const a[5];", False), + ("int a[5][6];", False), + ("const int a[5][6];", False), + ]: + p = Parser() + ast = _get_parser().parse(input) + decl = ast.children()[0][1] + node = decl.type + assert p._is_constant_globalvar(node) == expected_output diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py @@ -7,17 +7,20 @@ if sys.platform == 'win32': pass # no obvious -Werror equivalent on MSVC -elif (sys.platform == 'darwin' and - [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]): - pass # recent MacOSX come with clang by default, and passing some - # flags from the interpreter (-mno-fused-madd) generates a - # warning --- which is interpreted as an error with -Werror else: - # assume a standard GCC + if (sys.platform == 'darwin' and + [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]): + # special things for clang + extra_compile_args = [ + '-Werror', '-Qunused-arguments', '-Wno-error=shorten-64-to-32'] + else: + # assume a standard gcc + extra_compile_args = ['-Werror'] + class FFI(FFI): def verify(self, *args, **kwds): return super(FFI, self).verify( - *args, extra_compile_args=['-Werror'], **kwds) + *args, extra_compile_args=extra_compile_args, **kwds) def setup_module(): import cffi.verifier @@ -477,32 +480,71 @@ s = ffi.new("struct foo_s *") assert ffi.sizeof(s.a) == 17 * ffi.sizeof('int') -def test_struct_array_guess_length(): +def test_struct_array_no_length(): ffi = FFI() - ffi.cdef("struct foo_s { int a[]; ...; };") # <= no declared length - ffi.verify("struct foo_s { int x; int a[17]; int y; };") - assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int') - s = ffi.new("struct foo_s *") - assert ffi.sizeof(s.a) == 17 * ffi.sizeof('int') - -def test_struct_array_guess_length_2(): - ffi = FFI() - ffi.cdef("struct foo_s { int a[]; ...; };\n" # <= no declared length + ffi.cdef("struct foo_s { int a[]; int y; ...; };\n" "int bar(struct foo_s *);\n") lib = ffi.verify("struct foo_s { int x; int a[17]; int y; };\n" "int bar(struct foo_s *f) { return f->a[14]; }\n") assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int') s = ffi.new("struct foo_s *") + assert ffi.typeof(s.a) is ffi.typeof('int *') # because no length s.a[14] = 4242 assert lib.bar(s) == 4242 + # with no declared length, out-of-bound accesses are not detected + s.a[17] = -521 + assert s.y == s.a[17] == -521 + # + s = ffi.new("struct foo_s *", {'a': list(range(17))}) + assert s.a[16] == 16 + # overflows at construction time not detected either + s = ffi.new("struct foo_s *", {'a': list(range(18))}) + assert s.y == s.a[17] == 17 -def test_struct_array_guess_length_3(): +def test_struct_array_guess_length(): ffi = FFI() ffi.cdef("struct foo_s { int a[...]; };") ffi.verify("struct foo_s { int x; int a[17]; int y; };") assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int') s = ffi.new("struct foo_s *") assert ffi.sizeof(s.a) == 17 * ffi.sizeof('int') + py.test.raises(IndexError, 's.a[17]') + +def test_struct_array_c99_1(): + if sys.platform == 'win32': + py.test.skip("requires C99") + ffi = FFI() + ffi.cdef("struct foo_s { int x; int a[]; };") + ffi.verify("struct foo_s { int x; int a[]; };") + assert ffi.sizeof('struct foo_s') == 1 * ffi.sizeof('int') + s = ffi.new("struct foo_s *", [424242, 4]) + assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int') # the same in C + assert s.a[3] == 0 + s = ffi.new("struct foo_s *", [424242, [-40, -30, -20, -10]]) + assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int') + assert s.a[3] == -10 + s = ffi.new("struct foo_s *") + assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int') + s = ffi.new("struct foo_s *", [424242]) + assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int') + +def test_struct_array_c99_2(): + if sys.platform == 'win32': + py.test.skip("requires C99") + ffi = FFI() + ffi.cdef("struct foo_s { int x; int a[]; ...; };") + ffi.verify("struct foo_s { int x, y; int a[]; };") + assert ffi.sizeof('struct foo_s') == 2 * ffi.sizeof('int') + s = ffi.new("struct foo_s *", [424242, 4]) + assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int') + assert s.a[3] == 0 + s = ffi.new("struct foo_s *", [424242, [-40, -30, -20, -10]]) + assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int') + assert s.a[3] == -10 + s = ffi.new("struct foo_s *") + assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int') + s = ffi.new("struct foo_s *", [424242]) + assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int') def test_struct_ptr_to_array_field(): ffi = FFI() @@ -614,6 +656,21 @@ s.x = 17 assert s.x == 17 +def test_anonymous_enum(): + ffi = FFI() + ffi.cdef("enum { EE1 }; enum { EE2, EE3 };") + lib = ffi.verify("enum { EE1 }; enum { EE2, EE3 };") + assert lib.EE1 == 0 + assert lib.EE2 == 0 + assert lib.EE3 == 1 + +def test_nonfull_anonymous_enum(): + ffi = FFI() + ffi.cdef("enum { EE1, ... }; enum { EE3, ... };") + lib = ffi.verify("enum { EE2, EE1 }; enum { EE3 };") + assert lib.EE1 == 1 + assert lib.EE3 == 0 + def test_nonfull_enum_syntax2(): ffi = FFI() ffi.cdef("enum ee { EE1, EE2=\t..., EE3 };") @@ -1160,6 +1217,36 @@ ffi.cdef("union foo_u { char x; long *z; };") ffi.verify("union foo_u { char x; int y; long *z; };") +def test_ffi_union_partial(): + ffi = FFI() + ffi.cdef("union foo_u { char x; ...; };") + ffi.verify("union foo_u { char x; int y; };") + assert ffi.sizeof("union foo_u") == 4 + +def test_ffi_union_with_partial_struct(): + ffi = FFI() + ffi.cdef("struct foo_s { int x; ...; }; union foo_u { struct foo_s s; };") + ffi.verify("struct foo_s { int a; int x; }; " + "union foo_u { char b[32]; struct foo_s s; };") + assert ffi.sizeof("struct foo_s") == 8 + assert ffi.sizeof("union foo_u") == 32 + +def test_ffi_union_partial_2(): + ffi = FFI() + ffi.cdef("typedef union { char x; ...; } u1;") + ffi.verify("typedef union { char x; int y; } u1;") + assert ffi.sizeof("u1") == 4 + +def test_ffi_union_with_partial_struct_2(): + ffi = FFI() + ffi.cdef("typedef struct { int x; ...; } s1;" + "typedef union { s1 s; } u1;") + ffi.verify("typedef struct { int a; int x; } s1; " + "typedef union { char b[32]; s1 s; } u1;") + assert ffi.sizeof("s1") == 8 + assert ffi.sizeof("u1") == 32 + assert ffi.offsetof("u1", "s") == 0 + def test_ffi_struct_packed(): if sys.platform == 'win32': py.test.skip("needs a GCC extension") @@ -1423,7 +1510,12 @@ ffi = FFI() ffi.cdef("int fooarray[...];") lib = ffi.verify("int fooarray[50];") - assert repr(lib.fooarray).startswith("<cdata 'int *'") + assert repr(lib.fooarray).startswith("<cdata 'int[50]'") + +def test_bad_global_array_with_dotdotdot_length(): + ffi = FFI() + ffi.cdef("int fooarray[...];") + py.test.raises(VerificationError, ffi.verify, "char fooarray[23];") def test_struct_containing_struct(): ffi = FFI() @@ -1635,3 +1727,26 @@ #define FOO 42""") assert dir(lib) == ['AA', 'BB', 'FOO', 'somearray', 'somefunc', 'somevar', 'sv2'] + +def test_typeof_func_with_struct_argument(): + ffi = FFI() + ffi.cdef("""struct s { int a; }; int foo(struct s);""") + lib = ffi.verify("""struct s { int a; }; + int foo(struct s x) { return x.a; }""") + s = ffi.new("struct s *", [-1234]) + m = lib.foo(s[0]) + assert m == -1234 + assert repr(ffi.typeof(lib.foo)) == "<ctype 'int(*)(struct s)'>" + +def test_bug_const_char_ptr_array_1(): + ffi = FFI() + ffi.cdef("""const char *a[...];""") + lib = ffi.verify("""const char *a[5];""") + assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>" + +def test_bug_const_char_ptr_array_2(): + from cffi import FFI # ignore warnings + ffi = FFI() + ffi.cdef("""const int a[];""") + lib = ffi.verify("""const int a[5];""") + assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>" diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -2,7 +2,7 @@ import sys, os, imp, math, shutil import py from cffi import FFI, FFIError -from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffix +from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path from pypy.module.test_lib_pypy.cffi_tests.udir import udir @@ -250,7 +250,7 @@ lib = ffi.verify(csrc, force_generic_engine=self.generic, modulename=modname) assert lib.test1foo(143) == 80.0 - suffix = _get_so_suffix() + suffix = _get_so_suffixes()[0] fn1 = os.path.join(ffi.verifier.tmpdir, modname + '.c') fn2 = os.path.join(ffi.verifier.tmpdir, modname + suffix) assert ffi.verifier.sourcefilename == fn1 _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit