Author: Armin Rigo <ar...@tunes.org> Branch: qualtypes Changeset: r2282:ab9e37952de6 Date: 2015-09-30 11:00 +0200 http://bitbucket.org/cffi/cffi/changeset/ab9e37952de6/
Log: in-progress: add qualifiers through model.py, cparser.py, and recompiler.py diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -609,7 +609,7 @@ def make_accessor_locked(name): key = 'function ' + name if key in ffi._parser._declarations: - tp = ffi._parser._declarations[key] + tp, _ = ffi._parser._declarations[key] BType = ffi._get_cached_btype(tp) try: value = backendlib.load_function(BType, name) @@ -620,7 +620,7 @@ # key = 'variable ' + name if key in ffi._parser._declarations: - tp = ffi._parser._declarations[key] + tp, _ = ffi._parser._declarations[key] BType = ffi._get_cached_btype(tp) read_variable = backendlib.read_variable write_variable = backendlib.write_variable @@ -632,7 +632,7 @@ if not copied_enums: from . import model error = None - for key, tp in ffi._parser._declarations.items(): + for key, (tp, _) in ffi._parser._declarations.items(): if not isinstance(tp, model.EnumType): continue try: diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -192,6 +192,7 @@ if not decl.name: raise api.CDefError("typedef does not declare any name", decl) + quals = 0 if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and decl.type.type.names[-1] == '__dotdotdot__'): realtype = self._get_unknown_type(decl) @@ -202,8 +203,9 @@ decl.type.type.type.names == ['__dotdotdot__']): realtype = model.unknown_ptr_type(decl.name) else: - realtype = self._get_type(decl.type, name=decl.name) - self._declare('typedef ' + decl.name, realtype) + realtype, quals = self._get_type_and_quals( + decl.type, name=decl.name) + self._declare('typedef ' + decl.name, realtype, quals=quals) else: raise api.CDefError("unrecognized construct", decl) except api.FFIError as e: @@ -255,9 +257,9 @@ def _parse_decl(self, decl): node = decl.type if isinstance(node, pycparser.c_ast.FuncDecl): - tp = self._get_type(node, name=decl.name) + tp, quals = self._get_type_and_quals(node, name=decl.name) assert isinstance(tp, model.RawFunctionType) - tp = self._get_type_pointer(tp) + tp = self._get_type_pointer(tp, quals) self._declare('function ' + decl.name, tp) else: if isinstance(node, pycparser.c_ast.Struct): @@ -271,9 +273,10 @@ decl) # if decl.name: - tp = self._get_type(node, partial_length_ok=True) + tp, quals = self._get_type_and_quals(node, + partial_length_ok=True) if tp.is_raw_function: - tp = self._get_type_pointer(tp) + tp = self._get_type_pointer(tp, quals) self._declare('function ' + decl.name, tp) elif (tp.is_integer_type() and hasattr(decl, 'init') and @@ -287,10 +290,10 @@ _r_int_literal.match(decl.init.expr.value)): self._add_integer_constant(decl.name, '-' + decl.init.expr.value) - elif self._is_constant_globalvar(node): - self._declare('constant ' + decl.name, tp) + elif (quals & model.Q_CONST) and not tp.is_array_type: + self._declare('constant ' + decl.name, tp, quals=quals) else: - self._declare('variable ' + decl.name, tp) + self._declare('variable ' + decl.name, tp, quals=quals) def parse_type(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2] @@ -298,40 +301,51 @@ exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type(exprnode.type) + tp, quals = self._get_type_and_quals(exprnode.type) + return tp - def _declare(self, name, obj, included=False): + def _declare(self, name, obj, included=False, quals=0): if name in self._declarations: - if self._declarations[name] is obj: + prevobj, prevquals = self._declarations[name] + if prevobj is obj and prevquals == quals: return if not self._override: raise api.FFIError( "multiple declarations of %s (for interactive usage, " "try cdef(xx, override=True))" % (name,)) assert '__dotdotdot__' not in name.split() - self._declarations[name] = obj + self._declarations[name] = (obj, quals) if included: self._included_declarations.add(obj) - def _get_type_pointer(self, type, const=False, declname=None): + def _extract_quals(self, type): + quals = 0 + if isinstance(type, (pycparser.c_ast.TypeDecl, + pycparser.c_ast.PtrDecl)): + if 'const' in type.quals: + quals |= model.Q_CONST + if 'restrict' in type.quals: + quals |= model.Q_RESTRICT + return quals + + def _get_type_pointer(self, type, quals, declname=None): if isinstance(type, model.RawFunctionType): return type.as_function_pointer() if (isinstance(type, model.StructOrUnionOrEnum) and type.name.startswith('$') and type.name[1:].isdigit() and type.forcename is None and declname is not None): - return model.NamedPointerType(type, declname) - if const: - return model.ConstPointerType(type) - return model.PointerType(type) + return model.NamedPointerType(type, declname, quals) + return model.PointerType(type, quals) - def _get_type(self, typenode, name=None, partial_length_ok=False): + def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): - type = self._declarations['typedef ' + typenode.type.names[0]] - return type + tp, quals = self._declarations['typedef ' + typenode.type.names[0]] + quals |= self._extract_quals(typenode) + return tp, quals # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type @@ -340,18 +354,19 @@ else: length = self._parse_constant( typenode.dim, partial_length_ok=partial_length_ok) - tp = self._get_type(typenode.type, + tp, quals = self._get_type_and_quals(typenode.type, partial_length_ok=partial_length_ok) - return model.ArrayType(tp, length) + return model.ArrayType(tp, length), quals # if isinstance(typenode, pycparser.c_ast.PtrDecl): # pointer type - const = (isinstance(typenode.type, pycparser.c_ast.TypeDecl) - and 'const' in typenode.type.quals) - return self._get_type_pointer(self._get_type(typenode.type), const, - declname=name) + itemtype, itemquals = self._get_type_and_quals(typenode.type) + tp = self._get_type_pointer(itemtype, itemquals, declname=name) + quals = self._extract_quals(typenode) + return tp, quals # if isinstance(typenode, pycparser.c_ast.TypeDecl): + quals = self._extract_quals(typenode) type = typenode.type if isinstance(type, pycparser.c_ast.IdentifierType): # assume a primitive type. get it from .names, but reduce @@ -379,35 +394,38 @@ names = newnames + names ident = ' '.join(names) if ident == 'void': - return model.void_type + return model.void_type, quals if ident == '__dotdotdot__': raise api.FFIError(':%d: bad usage of "..."' % typenode.coord.line) - return resolve_common_type(ident) + return resolve_common_type(ident), quals # if isinstance(type, pycparser.c_ast.Struct): # 'struct foobar' - return self._get_struct_union_enum_type('struct', type, name) + tp = self._get_struct_union_enum_type('struct', type, name) + return tp, quals # if isinstance(type, pycparser.c_ast.Union): # 'union foobar' - return self._get_struct_union_enum_type('union', type, name) + tp = self._get_struct_union_enum_type('union', type, name) + return tp, quals # if isinstance(type, pycparser.c_ast.Enum): # 'enum foobar' - return self._get_struct_union_enum_type('enum', type, name) + tp = self._get_struct_union_enum_type('enum', type, name) + return tp, quals # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type - return self._parse_function_type(typenode, name) + return self._parse_function_type(typenode, name), 0 # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): return self._get_struct_union_enum_type('struct', typenode, name, - nested=True) + nested=True), 0 if isinstance(typenode, pycparser.c_ast.Union): return self._get_struct_union_enum_type('union', typenode, name, - nested=True) + nested=True), 0 # raise api.FFIError(":%d: bad or unsupported type declaration" % typenode.coord.line) @@ -426,28 +444,21 @@ raise api.CDefError( "%s: a function with only '(...)' as argument" " is not correct C" % (funcname or 'in expression')) - args = [self._as_func_arg(self._get_type(argdeclnode.type)) + args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type)) for argdeclnode in params] if not ellipsis and args == [model.void_type]: args = [] - result = self._get_type(typenode.type) + result, quals = self._get_type_and_quals(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) - def _as_func_arg(self, type): + def _as_func_arg(self, type, quals): if isinstance(type, model.ArrayType): - return model.PointerType(type.item) + return model.PointerType(type.item, quals) elif isinstance(type, model.RawFunctionType): return type.as_function_pointer() else: return type - def _is_constant_globalvar(self, typenode): - if isinstance(typenode, pycparser.c_ast.PtrDecl): - return 'const' in typenode.quals - if isinstance(typenode, pycparser.c_ast.TypeDecl): - return 'const' in typenode.quals - return False - def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): # First, a level of caching on the exact 'type' node of the AST. # This is obscure, but needed because pycparser "unrolls" declarations @@ -486,7 +497,7 @@ else: explicit_name = name key = '%s %s' % (kind, name) - tp = self._declarations.get(key, None) + tp, _ = self._declarations.get(key, (None, None)) # if tp is None: if kind == 'struct': @@ -528,6 +539,7 @@ fldnames = [] fldtypes = [] fldbitsize = [] + fldquals = [] for decl in type.decls: if (isinstance(decl.type, pycparser.c_ast.IdentifierType) and ''.join(decl.type.names) == '__dotdotdot__'): @@ -541,7 +553,8 @@ else: bitsize = self._parse_constant(decl.bitsize) self._partial_length = False - type = self._get_type(decl.type, partial_length_ok=True) + type, fqual = self._get_type_and_quals(decl.type, + partial_length_ok=True) if self._partial_length: self._make_partial(tp, nested) if isinstance(type, model.StructType) and type.partial: @@ -549,9 +562,11 @@ fldnames.append(decl.name or '') fldtypes.append(type) fldbitsize.append(bitsize) + fldquals.append(fqual) tp.fldnames = tuple(fldnames) tp.fldtypes = tuple(fldtypes) tp.fldbitsize = tuple(fldbitsize) + tp.fldquals = tuple(fldquals) if fldbitsize != [-1] * len(fldbitsize): if isinstance(tp, model.StructType) and tp.partial: raise NotImplementedError("%s: using both bitfields and '...;'" @@ -632,14 +647,12 @@ return tp def include(self, other): - for name, tp in other._declarations.items(): + for name, (tp, quals) in other._declarations.items(): if name.startswith('anonymous $enum_$'): continue # fix for test_anonymous_enum_include kind = name.split(' ', 1)[0] - if kind in ('struct', 'union', 'enum', 'anonymous'): - self._declare(name, tp, included=True) - elif kind == 'typedef': - self._declare(name, tp, included=True) + if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef'): + self._declare(name, tp, included=True, quals=quals) for k, v in other._int_constants.items(): self._add_constants(k, v) diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -4,6 +4,18 @@ from .lock import allocate_lock +# type qualifiers +Q_CONST = 0x01 +Q_RESTRICT = 0x02 + +def qualify(quals, replace_with): + if quals & Q_CONST: + replace_with = ' const ' + replace_with.lstrip() + if quals & Q_RESTRICT: + replace_with = ' restrict ' + replace_with.lstrip() + return replace_with + + class BaseTypeByIdentity(object): is_array_type = False is_raw_function = False @@ -225,16 +237,14 @@ class PointerType(BaseType): - _attrs_ = ('totype',) - _base_pattern = " *&" - _base_pattern_array = "(*&)" + _attrs_ = ('totype', 'quals') - def __init__(self, totype): + def __init__(self, totype, quals=0): self.totype = totype + self.quals = quals + extra = qualify(quals, " *&") if totype.is_array_type: - extra = self._base_pattern_array - else: - extra = self._base_pattern + extra = "(%s)" % (extra.lstrip(),) self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra) def build_backend_type(self, ffi, finishlist): @@ -243,10 +253,8 @@ voidp_type = PointerType(void_type) - -class ConstPointerType(PointerType): - _base_pattern = " const *&" - _base_pattern_array = "(const *&)" +def ConstPointerType(totype): + return PointerType(totype, Q_CONST) const_voidp_type = ConstPointerType(void_type) @@ -254,8 +262,8 @@ class NamedPointerType(PointerType): _attrs_ = ('totype', 'name') - def __init__(self, totype, name): - PointerType.__init__(self, totype) + def __init__(self, totype, name, quals=0): + PointerType.__init__(self, totype, quals) self.name = name self.c_name_with_marker = name + '&' @@ -315,11 +323,12 @@ partial = False packed = False - def __init__(self, name, fldnames, fldtypes, fldbitsize): + def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None): self.name = name self.fldnames = fldnames self.fldtypes = fldtypes self.fldbitsize = fldbitsize + self.fldquals = fldquals self.build_c_name_with_marker() def has_anonymous_struct_fields(self): @@ -331,14 +340,17 @@ return False def enumfields(self): - for name, type, bitsize in zip(self.fldnames, self.fldtypes, - self.fldbitsize): + fldquals = self.fldquals + if fldquals is None: + fldquals = (0,) * len(self.fldnames) + for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, + self.fldbitsize, fldquals): if name == '' and isinstance(type, StructOrUnion): # nested anonymous struct/union for result in type.enumfields(): yield result else: - yield (name, type, bitsize) + yield (name, type, bitsize, quals) def force_flatten(self): # force the struct or union to have a declaration that lists @@ -347,13 +359,16 @@ names = [] types = [] bitsizes = [] - for name, type, bitsize in self.enumfields(): + fldquals = [] + for name, type, bitsize, quals in self.enumfields(): names.append(name) types.append(type) bitsizes.append(bitsize) + fldquals.append(quals) self.fldnames = tuple(names) self.fldtypes = tuple(types) self.fldbitsize = tuple(bitsizes) + self.fldquals = tuple(fldquals) def get_cached_btype(self, ffi, finishlist, can_delay=False): BType = StructOrUnionOrEnum.get_cached_btype(self, ffi, finishlist, diff --git a/cffi/recompiler.py b/cffi/recompiler.py --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -195,17 +195,16 @@ elif isinstance(tp, model.StructOrUnion): if tp.fldtypes is not None and ( tp not in self.ffi._parser._included_declarations): - for name1, tp1, _ in tp.enumfields(): + for name1, tp1, _, _ in tp.enumfields(): self._do_collect_type(self._field_type(tp, name1, tp1)) else: for _, x in tp._get_items(): self._do_collect_type(x) - def _get_declarations(self): - return sorted(self.ffi._parser._declarations.items()) - def _generate(self, step_name): - for name, tp in self._get_declarations(): + lst = self.ffi._parser._declarations.items() + lst.sort() + for name, (tp, quals) in lst: kind, realname = name.split(' ', 1) try: method = getattr(self, '_generate_cpy_%s_%s' % (kind, @@ -214,6 +213,7 @@ raise ffiplatform.VerificationError( "not implemented in recompile(): %r" % name) try: + self._current_quals = quals method(tp, realname) except Exception as e: model.attach_exception_info(e, name) @@ -774,7 +774,7 @@ prnt('{') prnt(' /* only to generate compile-time warnings or errors */') prnt(' (void)p;') - for fname, ftype, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize, fqual in tp.enumfields(): try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double @@ -788,8 +788,9 @@ and (ftype.length is None or ftype.length == '...')): ftype = ftype.item fname = fname + '[0]' + tmp = model.qualify(fqual, '*tmp') prnt(' { %s = &p->%s; (void)tmp; }' % ( - ftype.get_c_name('*tmp', 'field %r'%fname), fname)) + ftype.get_c_name(tmp, 'field %r'%fname), fname)) except ffiplatform.VerificationError as e: prnt(' /* %s */' % str(e)) # cannot verify it, ignore prnt('}') @@ -823,7 +824,7 @@ c_fields = [] if reason_for_not_expanding is None: enumfields = list(tp.enumfields()) - for fldname, fldtype, fbitsize in enumfields: + for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) # cname is None for _add_missing_struct_unions() only op = OP_NOOP @@ -1085,7 +1086,9 @@ # if 'tp' were a function type, but that is not possible here. # (If 'tp' is a function _pointer_ type, then casts from "fn_t # **" to "void *" are again no-ops, as far as I can tell.) - prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,))) + decl = '*_cffi_var_%s(void)' % (name,) + decl = model.qualify(self._current_quals, decl) + prnt('static ' + tp.get_c_name(decl)) prnt('{') prnt(' return %s(%s);' % (ampersand, name)) prnt('}') diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py --- a/cffi/vengine_cpy.py +++ b/cffi/vengine_cpy.py @@ -197,7 +197,10 @@ return library def _get_declarations(self): - return sorted(self.ffi._parser._declarations.items()) + lst = [(key, tp) for (key, (tp, qual)) in + self.ffi._parser._declarations.items()] + lst.sort() + return lst def _generate(self, step_name): for name, tp in self._get_declarations(): @@ -468,7 +471,7 @@ prnt('{') prnt(' /* only to generate compile-time warnings or errors */') prnt(' (void)p;') - for fname, ftype, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize, fqual in tp.enumfields(): if (isinstance(ftype, model.PrimitiveType) and ftype.is_integer_type()) or fbitsize >= 0: # accept all integers, but complain on float or double @@ -488,7 +491,7 @@ prnt(' static Py_ssize_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, ftype, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize, fqual in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) @@ -552,7 +555,7 @@ check(layout[0], ffi.sizeof(BStruct), "wrong total size") check(layout[1], ffi.alignof(BStruct), "wrong total alignment") i = 2 - for fname, ftype, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize, fqual in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py --- a/cffi/vengine_gen.py +++ b/cffi/vengine_gen.py @@ -87,7 +87,10 @@ return library def _get_declarations(self): - return sorted(self.ffi._parser._declarations.items()) + lst = [(key, tp) for (key, (tp, qual)) in + self.ffi._parser._declarations.items()] + lst.sort() + return lst def _generate(self, step_name): for name, tp in self._get_declarations(): @@ -260,7 +263,7 @@ prnt('{') prnt(' /* only to generate compile-time warnings or errors */') prnt(' (void)p;') - for fname, ftype, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize, fqual in tp.enumfields(): if (isinstance(ftype, model.PrimitiveType) and ftype.is_integer_type()) or fbitsize >= 0: # accept all integers, but complain on float or double @@ -280,7 +283,7 @@ prnt(' static intptr_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') - for fname, ftype, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize, fqual in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now prnt(' offsetof(%s, %s),' % (cname, fname)) @@ -342,7 +345,7 @@ check(layout[0], ffi.sizeof(BStruct), "wrong total size") check(layout[1], ffi.alignof(BStruct), "wrong total alignment") i = 2 - for fname, ftype, fbitsize in tp.enumfields(): + for fname, ftype, fbitsize, fqual in tp.enumfields(): if fbitsize >= 0: continue # xxx ignore fbitsize for now check(layout[i], ffi.offsetof(BStruct, fname), diff --git a/testing/cffi0/test_model.py b/testing/cffi0/test_model.py --- a/testing/cffi0/test_model.py +++ b/testing/cffi0/test_model.py @@ -57,6 +57,11 @@ assert ptr_type.get_c_name("") == "int(const *)[5]" assert ptr_type.get_c_name("*x") == "int(const * *x)[5]" +def test_qual_pointer_type(): + ptr_type = PointerType(PrimitiveType("long long"), Q_RESTRICT) + assert ptr_type.get_c_name("") == "long long restrict *" + assert const_voidp_type.get_c_name("") == "void const *" + def test_unknown_pointer_type(): ptr_type = unknown_ptr_type("foo_p") assert ptr_type.get_c_name("") == "foo_p" diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py --- a/testing/cffi0/test_parsing.py +++ b/testing/cffi0/test_parsing.py @@ -307,7 +307,6 @@ ffi.cdef("void f(WPARAM);") def test__is_constant_globalvar(): - from cffi.cparser import Parser, _get_parser for input, expected_output in [ ("int a;", False), ("const int a;", True), @@ -324,11 +323,23 @@ ("int a[5][6];", False), ("const int a[5][6];", False), ]: - p = Parser() - ast = _get_parser().parse(input) - decl = ast.children()[0][1] - node = decl.type - assert p._is_constant_globalvar(node) == expected_output + ffi = FFI() + ffi.cdef(input) + declarations = ffi._parser._declarations + assert ('constant a' in declarations) == expected_output + assert ('variable a' in declarations) == (not expected_output) + +def test_restrict(): + from cffi import model + for input, expected_output in [ + ("int a;", False), + ("restrict int a;", True), + ("int *a;", False), + ]: + ffi = FFI() + ffi.cdef(input) + tp, quals = ffi._parser._declarations['variable a'] + assert bool(quals & model.Q_RESTRICT) == expected_output def test_enum(): ffi = FFI() diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py --- a/testing/cffi1/test_verify1.py +++ b/testing/cffi1/test_verify1.py @@ -1922,7 +1922,7 @@ assert repr(ffi.typeof(lib.a)) == "<ctype 'char *[5]'>" def test_bug_const_char_ptr_array_2(): - ffi = FFI_warnings_not_error() # ignore warnings + ffi = FFI() ffi.cdef("""const int a[];""") lib = ffi.verify("""const int a[5];""") assert repr(ffi.typeof(lib.a)) == "<ctype 'int *'>" _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit