Author: mattip <matti.pi...@gmail.com> Branch: numpy-1.10 Changeset: r80686:9286403425c3 Date: 2015-11-15 18:07 +0200 http://bitbucket.org/pypy/pypy/changeset/9286403425c3/
Log: merge default into branch diff too long, truncating to 2000 out of 4464 lines diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.3.0 +Version: 1.3.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.3.0" -__version_info__ = (1, 3, 0) +__version__ = "1.3.1" +__version_info__ = (1, 3, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -514,12 +514,17 @@ if self.baseinttype is not None: return self.baseinttype.get_cached_btype(ffi, finishlist) # + from . import api if self.enumvalues: smallest_value = min(self.enumvalues) largest_value = max(self.enumvalues) else: - smallest_value = 0 - largest_value = 0 + import warnings + warnings.warn("%r has no values explicitly defined; next version " + "will refuse to guess which integer type it is " + "meant to be (unsigned/signed, int/long)" + % self._get_c_name()) + smallest_value = largest_value = 0 if smallest_value < 0: # needs a signed type sign = 1 candidate1 = PrimitiveType("int") diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -29,3 +29,7 @@ Support common use-cases for __array_interface__, passes upstream tests +.. branch: no-class-specialize + +Some refactoring of class handling in the annotator. +Remove class specialisation and _settled_ flag. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -28,7 +28,6 @@ """This is the abstract root class of all wrapped objects that live in a 'normal' object space like StdObjSpace.""" __slots__ = () - _settled_ = True user_overridden_class = False def getdict(self, space): diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload, clibffi -VERSION = "1.3.0" +VERSION = "1.3.1" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.3.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.3.1", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -13,7 +13,7 @@ def __init__(self, msg): self.msg = msg -from rpython.annotator.description import FORCE_ATTRIBUTES_INTO_CLASSES +from rpython.annotator.classdesc import FORCE_ATTRIBUTES_INTO_CLASSES from rpython.annotator.model import SomeString # this is necessary due to annmixlevel diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py --- a/pypy/module/_multiprocessing/interp_win32.py +++ b/pypy/module/_multiprocessing/interp_win32.py @@ -17,7 +17,7 @@ NMPWAIT_WAIT_FOREVER ERROR_PIPE_CONNECTED ERROR_SEM_TIMEOUT ERROR_PIPE_BUSY ERROR_NO_SYSTEM_RESOURCES ERROR_BROKEN_PIPE ERROR_MORE_DATA - ERROR_ALREADY_EXISTS + ERROR_ALREADY_EXISTS ERROR_NO_DATA """.split() class CConfig: diff --git a/pypy/module/cpyext/pystrtod.py b/pypy/module/cpyext/pystrtod.py --- a/pypy/module/cpyext/pystrtod.py +++ b/pypy/module/cpyext/pystrtod.py @@ -5,10 +5,23 @@ from rpython.rlib import rdtoa from rpython.rlib import rfloat from rpython.rlib import rposix, jit +from rpython.rlib.rarithmetic import intmask from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem import rffi +# PyOS_double_to_string's "type", if non-NULL, will be set to one of: +Py_DTST_FINITE = 0 +Py_DTST_INFINITE = 1 +Py_DTST_NAN = 2 + +# Match the "type" back to values in CPython +DOUBLE_TO_STRING_TYPES_MAP = { + rfloat.DIST_FINITE: Py_DTST_FINITE, + rfloat.DIST_INFINITY: Py_DTST_INFINITE, + rfloat.DIST_NAN: Py_DTST_NAN +} + @cpython_api([rffi.CCHARP, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) @jit.dont_look_inside # direct use of _get_errno() def PyOS_string_to_double(space, s, endptr, w_overflow_exception): @@ -68,3 +81,42 @@ finally: if not user_endptr: lltype.free(endptr, flavor='raw') + +@cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP) +def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): + """Convert a double val to a string using supplied + format_code, precision, and flags. + + format_code must be one of 'e', 'E', 'f', 'F', + 'g', 'G' or 'r'. For 'r', the supplied precision + must be 0 and is ignored. The 'r' format code specifies the + standard repr() format. + + flags can be zero or more of the values Py_DTSF_SIGN, + Py_DTSF_ADD_DOT_0, or Py_DTSF_ALT, or-ed together: + + Py_DTSF_SIGN means to always precede the returned string with a sign + character, even if val is non-negative. + + Py_DTSF_ADD_DOT_0 means to ensure that the returned string will not look + like an integer. + + Py_DTSF_ALT means to apply "alternate" formatting rules. See the + documentation for the PyOS_snprintf() '#' specifier for + details. + + If ptype is non-NULL, then the value it points to will be set to one of + Py_DTST_FINITE, Py_DTST_INFINITE, or Py_DTST_NAN, signifying that + val is a finite number, an infinite number, or not a number, respectively. + + The return value is a pointer to buffer with the converted string or + NULL if the conversion failed. The caller is responsible for freeing the + returned string by calling PyMem_Free(). + """ + buffer, rtype = rfloat.double_to_string(val, format_code, + intmask(precision), + intmask(flags)) + if ptype != lltype.nullptr(rffi.INTP.TO): + ptype[0] = rffi.cast(rffi.INT, DOUBLE_TO_STRING_TYPES_MAP[rtype]) + bufp = rffi.str2charp(buffer) + return bufp diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test/test_pystrtod.py --- a/pypy/module/cpyext/test/test_pystrtod.py +++ b/pypy/module/cpyext/test/test_pystrtod.py @@ -1,5 +1,6 @@ import math +from pypy.module.cpyext import pystrtod from pypy.module.cpyext.test.test_api import BaseApiTest from rpython.rtyper.lltypesystem import rffi from rpython.rtyper.lltypesystem import lltype @@ -91,3 +92,76 @@ api.PyErr_Clear() rffi.free_charp(s) lltype.free(endp, flavor='raw') + + +class TestPyOS_double_to_string(BaseApiTest): + + def test_format_code(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(150.0, 'e', 1, 0, ptype) + assert '1.5e+02' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_precision(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(3.14159269397, 'g', 5, 0, ptype) + assert '3.1416' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_flags_sign(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(-3.14, 'g', 3, 1, ptype) + assert '-3.14' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_flags_add_dot_0(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(3, 'g', 5, 2, ptype) + assert '3.0' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_flags_alt(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(314., 'g', 3, 4, ptype) + assert '314.' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_ptype_nan(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(float('nan'), 'g', 3, 4, ptype) + assert 'nan' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_NAN == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_ptype_infinity(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(1e200 * 1e200, 'g', 0, 0, ptype) + assert 'inf' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_INFINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_ptype_null(self, api): + ptype = lltype.nullptr(rffi.INTP.TO) + r = api.PyOS_double_to_string(3.14, 'g', 3, 0, ptype) + assert '3.14' == rffi.charp2str(r) + assert ptype == lltype.nullptr(rffi.INTP.TO) + rffi.free_charp(r) \ No newline at end of file diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py --- a/pypy/module/marshal/interp_marshal.py +++ b/pypy/module/marshal/interp_marshal.py @@ -156,9 +156,6 @@ put_tuple_w(TYPE, tuple_w) puts tuple_w, an unwrapped list of wrapped objects """ - # _annspecialcase_ = "specialize:ctr_location" # polymorphic - # does not work with subclassing - def __init__(self, space, writer, version): self.space = space ## self.put = putfunc diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -684,8 +684,9 @@ arr_iter, arr_state = arr.create_iter() arr_dtype = arr.get_dtype() index_dtype = index.get_dtype() - # XXX length of shape of index as well? - while not index_iter.done(index_state): + # support the deprecated form where arr([True]) will return arr[0, ...] + # by iterating over res_iter, not index_iter + while not res_iter.done(res_state): getitem_filter_driver.jit_merge_point(shapelen=shapelen, index_dtype=index_dtype, arr_dtype=arr_dtype, diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2235,6 +2235,9 @@ c = array([True,False,True],bool) b = a[c] assert (a[c] == [[1, 2, 3], [7, 8, 9]]).all() + c = array([True]) + b = a[c] + assert b.shape == (1, 3) def test_bool_array_index_setitem(self): from numpy import arange, array diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1336,7 +1336,8 @@ # these depend on user-defined data, so should not be shared assert ffi1.typeof("struct foo") is not ffi2.typeof("struct foo") assert ffi1.typeof("union foo *") is not ffi2.typeof("union foo*") - assert ffi1.typeof("enum foo") is not ffi2.typeof("enum foo") + # the following test is an opaque enum, which we no longer support + #assert ffi1.typeof("enum foo") is not ffi2.typeof("enum foo") # sanity check: twice 'ffi1' assert ffi1.typeof("struct foo*") is ffi1.typeof("struct foo *") @@ -1348,6 +1349,17 @@ assert ffi.getctype("pe") == 'e *' assert ffi.getctype("e1*") == 'e1 *' + def test_opaque_enum(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef("enum foo;") + from cffi import __version_info__ + if __version_info__ < (1, 4): + py.test.skip("re-enable me in version 1.4") + e = py.test.raises(CDefError, ffi.cast, "enum foo", -1) + assert str(e.value) == ( + "'enum foo' has no values explicitly defined: refusing to guess " + "which integer type it is meant to be (unsigned/signed, int/long)") + def test_new_ctype(self): ffi = FFI(backend=self.Backend()) p = ffi.new("int *") diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -1231,6 +1231,21 @@ def setitem(self, index, char): self.data[index] = char + def getslice(self, start, stop, step, size): + if size == 0: + return "" + if step == 1: + assert 0 <= start <= stop + if start == 0 and stop == len(self.data): + return "".join(self.data) + return "".join(self.data[start:stop]) + return Buffer.getslice(self, start, stop, step, size) + + def setslice(self, start, string): + # No bounds checks. + for i in range(len(string)): + self.data[start + i] = string[i] + @specialize.argtype(1) def _memcmp(selfvalue, buffer, length): diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py --- a/pypy/objspace/std/test/test_tupleobject.py +++ b/pypy/objspace/std/test/test_tupleobject.py @@ -413,8 +413,9 @@ from __pypy__ import specialized_zip_2_lists except ImportError: specialized_zip_2_lists = zip - raises(TypeError, specialized_zip_2_lists, [], ()) - raises(TypeError, specialized_zip_2_lists, (), []) + else: + raises(TypeError, specialized_zip_2_lists, [], ()) + raises(TypeError, specialized_zip_2_lists, (), []) assert specialized_zip_2_lists([], []) == [ ] assert specialized_zip_2_lists([2, 3], []) == [ diff --git a/pypy/tool/ann_override.py b/pypy/tool/ann_override.py --- a/pypy/tool/ann_override.py +++ b/pypy/tool/ann_override.py @@ -2,6 +2,7 @@ from rpython.annotator.policy import AnnotatorPolicy from rpython.flowspace.model import Constant from rpython.annotator import specialize +from rpython.annotator.classdesc import InstanceSource, ClassDef @@ -20,7 +21,6 @@ def specialize__wrap(self, funcdesc, args_s): from pypy.interpreter.baseobjspace import W_Root - from rpython.annotator.classdef import ClassDef W_Root_def = funcdesc.bookkeeper.getuniqueclassdef(W_Root) typ = args_s[1].knowntype if isinstance(typ, ClassDef): @@ -50,54 +50,34 @@ typ = (None, str) return funcdesc.cachedgraph(typ) - def _remember_immutable(self, t, cached): - # for jit benefit - if cached not in t._immutable_fields_: # accessed this way just - # for convenience - t._immutable_fields_.append(cached) - - def attach_lookup(self, t, attr): - cached = "cached_%s" % attr - if not t.is_heaptype() and not t.is_cpytype(): - self._remember_immutable(t, cached) - setattr(t, cached, t._lookup(attr)) - return True - return False - - def attach_lookup_in_type_where(self, t, attr): - cached = "cached_where_%s" % attr - if not t.is_heaptype() and not t.is_cpytype(): - self._remember_immutable(t, cached) - setattr(t, cached, t._lookup_where(attr)) - return True - return False - def consider_lookup(self, bookkeeper, attr): - from rpython.annotator.classdef import InstanceSource assert attr not in self.lookups from pypy.objspace.std import typeobject cached = "cached_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc + classdesc.immutable_fields.add(cached) classdesc.classdict[cached] = Constant(None) clsdef.add_source_for_attribute(cached, classdesc) for t in self.pypytypes: - if self.attach_lookup(t, attr): + if not (t.is_heaptype() or t.is_cpytype()): + setattr(t, cached, t._lookup(attr)) source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) self.lookups[attr] = True def consider_lookup_in_type_where(self, bookkeeper, attr): - from rpython.annotator.classdef import InstanceSource assert attr not in self.lookups_where from pypy.objspace.std import typeobject cached = "cached_where_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc + classdesc.immutable_fields.add(cached) classdesc.classdict[cached] = Constant((None, None)) clsdef.add_source_for_attribute(cached, classdesc) for t in self.pypytypes: - if self.attach_lookup_in_type_where(t, attr): + if not (t.is_heaptype() or t.is_cpytype()): + setattr(t, cached, t._lookup_where(attr)) source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) self.lookups_where[attr] = True @@ -135,18 +115,19 @@ def event(self, bookkeeper, what, x): from pypy.objspace.std import typeobject if isinstance(x, typeobject.W_TypeObject): - from rpython.annotator.classdef import InstanceSource clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) self.pypytypes[x] = True #print "TYPE", x for attr in self.lookups: - if attr and self.attach_lookup(x, attr): + if attr and not (x.is_heaptype() or x.is_cpytype()): cached = "cached_%s" % attr + setattr(x, cached, x._lookup(attr)) source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) for attr in self.lookups_where: - if attr and self.attach_lookup_in_type_where(x, attr): + if attr and not (x.is_heaptype() or x.is_cpytype()): cached = "cached_where_%s" % attr + setattr(x, cached, x._lookup_where(attr)) source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) return diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -14,8 +14,8 @@ SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint, s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked, - SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty, AnnotatorError) -from rpython.annotator.classdef import InstanceSource, ClassDef + SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty) +from rpython.annotator.classdesc import ClassDef, ClassDesc from rpython.annotator.listdef import ListDef, ListItem from rpython.annotator.dictdef import DictDef from rpython.annotator import description @@ -23,7 +23,6 @@ from rpython.annotator.argument import simple_args from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic from rpython.tool.algo.unionfind import UnionFind -from rpython.tool.flattenrec import FlattenRecursion from rpython.rtyper import extregistry @@ -163,9 +162,7 @@ s_callable.consider_call_site(args, s_result, call_op) def getuniqueclassdef(self, cls): - """Get the ClassDef associated with the given user cls. - Avoid using this! It breaks for classes that must be specialized. - """ + """Get the ClassDef associated with the given user cls.""" assert cls is not object desc = self.getdesc(cls) return desc.getuniqueclassdef() @@ -334,8 +331,9 @@ and x.__class__.__module__ != '__builtin__': if hasattr(x, '_cleanup_'): x._cleanup_() - self.see_mutable(x) - result = SomeInstance(self.getuniqueclassdef(x.__class__)) + classdef = self.getuniqueclassdef(x.__class__) + classdef.see_instance(x) + result = SomeInstance(classdef) elif x is None: return s_None else: @@ -362,7 +360,7 @@ if pyobj.__module__ == '__builtin__': # avoid making classdefs for builtin types result = self.getfrozen(pyobj) else: - result = description.ClassDesc(self, pyobj) + result = ClassDesc(self, pyobj) elif isinstance(pyobj, types.MethodType): if pyobj.im_self is None: # unbound return self.getdesc(pyobj.im_func) @@ -375,11 +373,11 @@ self.getdesc(pyobj.im_self)) # frozendesc else: # regular method origincls, name = origin_of_meth(pyobj) - self.see_mutable(pyobj.im_self) + classdef = self.getuniqueclassdef(pyobj.im_class) + classdef.see_instance(pyobj.im_self) assert pyobj == getattr(pyobj.im_self, name), ( "%r is not %s.%s ??" % (pyobj, pyobj.im_self, name)) # emulate a getattr to make sure it's on the classdef - classdef = self.getuniqueclassdef(pyobj.im_class) classdef.find_attribute(name) result = self.getmethoddesc( self.getdesc(pyobj.im_func), # funcdesc @@ -400,15 +398,6 @@ self.descs[pyobj] = result return result - def have_seen(self, x): - # this might need to expand some more. - if x in self.descs: - return True - elif (x.__class__, x) in self.seen_mutable: - return True - else: - return False - def getfrozen(self, pyobj): return description.FrozenDesc(self, pyobj) @@ -425,22 +414,6 @@ self.methoddescs[key] = result return result - _see_mutable_flattenrec = FlattenRecursion() - - def see_mutable(self, x): - key = (x.__class__, x) - if key in self.seen_mutable: - return - clsdef = self.getuniqueclassdef(x.__class__) - self.seen_mutable[key] = True - self.event('mutable', x) - source = InstanceSource(self, x) - def delayed(): - for attr in source.all_instance_attributes(): - clsdef.add_source_for_attribute(attr, source) - # ^^^ can trigger reflowing - self._see_mutable_flattenrec(delayed) - def valueoftype(self, t): return annotationoftype(t, self) @@ -495,6 +468,20 @@ return s_result + def getattr_locations(self, clsdesc, attrname): + attrdef = clsdesc.classdef.find_attribute(attrname) + return attrdef.read_locations + + def record_getattr(self, clsdesc, attrname): + locations = self.getattr_locations(clsdesc, attrname) + locations.add(self.position_key) + + def update_attr(self, clsdef, attrdef): + locations = self.getattr_locations(clsdef.classdesc, attrdef.name) + for position in locations: + self.annotator.reflowfromposition(position) + attrdef.validate(homedef=clsdef) + def pbc_call(self, pbc, args, emulated=None): """Analyse a call to a SomePBC() with the given args (list of annotations). diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -5,13 +5,14 @@ from collections import OrderedDict from rpython.annotator.model import ( - SomeInteger, SomeObject, SomeChar, SomeBool, SomeString, SomeTuple, + SomeInteger, SomeChar, SomeBool, SomeString, SomeTuple, SomeUnicodeCodePoint, SomeFloat, unionof, SomeUnicodeString, SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator, SomeOrderedDict, SomeByteArray, add_knowntypedata, s_ImpossibleValue,) from rpython.annotator.bookkeeper import ( getbookkeeper, immutablevalue, BUILTIN_ANALYZERS, analyzer_for) from rpython.annotator import description +from rpython.annotator.classdesc import ClassDef from rpython.flowspace.model import Constant import rpython.rlib.rarithmetic import rpython.rlib.objectmodel @@ -124,7 +125,6 @@ def our_issubclass(cls1, cls2): """ we're going to try to be less silly in the face of old-style classes""" - from rpython.annotator.classdef import ClassDef if cls2 is object: return True def classify(cls): diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py deleted file mode 100644 --- a/rpython/annotator/classdef.py +++ /dev/null @@ -1,434 +0,0 @@ -""" -Type inference for user-defined classes. -""" -from rpython.annotator.model import ( - SomePBC, s_ImpossibleValue, unionof, s_None, AnnotatorError) -from rpython.annotator import description - - -# The main purpose of a ClassDef is to collect information about class/instance -# attributes as they are really used. An Attribute object is stored in the -# most general ClassDef where an attribute of that name is read/written: -# classdef.attrs = {'attrname': Attribute()} -# -# The following invariants hold: -# -# (A) if an attribute is read/written on an instance of class A, then the -# classdef of A or a parent class of A has an Attribute object corresponding -# to that name. -# -# (I) if B is a subclass of A, then they don't both have an Attribute for the -# same name. (All information from B's Attribute must be merged into A's.) -# -# Additionally, each ClassDef records an 'attr_sources': it maps attribute names -# to a list of 'source' objects that want to provide a constant value for this -# attribute at the level of this class. The attr_sources provide information -# higher in the class hierarchy than concrete Attribute()s. It is for the case -# where (so far or definitely) the user program only reads/writes the attribute -# at the level of a subclass, but a value for this attribute could possibly -# exist in the parent class or in an instance of a parent class. -# -# The point of not automatically forcing the Attribute instance up to the -# parent class which has a class attribute of the same name is apparent with -# multiple subclasses: -# -# A -# attr=s1 -# / \ -# / \ -# B C -# attr=s2 attr=s3 -# -# XXX this does not seem to be correct, but I don't know how to phrase -# it correctly. See test_specific_attributes in test_annrpython -# -# In this case, as long as 'attr' is only read/written from B or C, the -# Attribute on B says that it can be 's1 or s2', and the Attribute on C says -# it can be 's1 or s3'. Merging them into a single Attribute on A would give -# the more imprecise 's1 or s2 or s3'. -# -# The following invariant holds: -# -# (II) if a class A has an Attribute, the 'attr_sources' for the same name is -# empty. It is also empty on all subclasses of A. (The information goes -# into the Attribute directly in this case.) -# -# The following invariant holds: -# -# (III) for a class A, each attrsource that comes from the class (as opposed to -# from a prebuilt instance) must be merged into all Attributes of the -# same name in all subclasses of A, if any. (Parent class attributes can -# be visible in reads from instances of subclasses.) - -class Attribute(object): - # readonly-ness - # SomeThing-ness - # NB. an attribute is readonly if it is a constant class attribute. - # Both writing to the instance attribute and discovering prebuilt - # instances that have the attribute set will turn off readonly-ness. - - def __init__(self, name, bookkeeper): - assert name != '__class__' - self.name = name - self.bookkeeper = bookkeeper - self.s_value = s_ImpossibleValue - self.readonly = True - self.attr_allowed = True - self.read_locations = {} - - def add_constant_source(self, classdef, source): - s_value = source.s_get_value(classdef, self.name) - if source.instance_level: - # a prebuilt instance source forces readonly=False, see above - self.modified(classdef) - s_new_value = unionof(self.s_value, s_value) # XXX "source %r attr %s" % (source, self.name), - self.s_value = s_new_value - - def getvalue(self): - # Same as 'self.s_value' for historical reasons. - return self.s_value - - def merge(self, other, classdef='?'): - assert self.name == other.name - s_new_value = unionof(self.s_value, other.s_value) # XXX "%s attr %s" % (classdef, self.name) - self.s_value = s_new_value - if not other.readonly: - self.modified(classdef) - self.read_locations.update(other.read_locations) - - def mutated(self, homedef): # reflow from attr read positions - s_newvalue = self.getvalue() - - for position in self.read_locations: - self.bookkeeper.annotator.reflowfromposition(position) - - # check for method demotion and after-the-fact method additions - if isinstance(s_newvalue, SomePBC): - attr = self.name - if s_newvalue.getKind() == description.MethodDesc: - # is method - if homedef.classdesc.read_attribute(attr, None) is None: - if not homedef.check_missing_attribute_update(attr): - for desc in s_newvalue.descriptions: - if desc.selfclassdef is None: - if homedef.classdesc.settled: - raise AnnotatorError( - "demoting method %s to settled class " - "%s not allowed" % (self.name, homedef) - ) - break - - # check for attributes forbidden by slots or _attrs_ - if homedef.classdesc.all_enforced_attrs is not None: - if self.name not in homedef.classdesc.all_enforced_attrs: - self.attr_allowed = False - if not self.readonly: - raise NoSuchAttrError( - "the attribute %r goes here to %r, " - "but it is forbidden here" % ( - self.name, homedef)) - - def modified(self, classdef='?'): - self.readonly = False - if not self.attr_allowed: - raise NoSuchAttrError( - "Attribute %r on %r should be read-only.\n" % (self.name, - classdef) + - "This error can be caused by another 'getattr' that promoted\n" - "the attribute here; the list of read locations is:\n" + - '\n'.join([str(loc[0]) for loc in self.read_locations])) - -class ClassDef(object): - "Wraps a user class." - - def __init__(self, bookkeeper, classdesc): - self.bookkeeper = bookkeeper - self.attrs = {} # {name: Attribute} - self.classdesc = classdesc - self.name = self.classdesc.name - self.shortname = self.name.split('.')[-1] - self.subdefs = [] - self.attr_sources = {} # {name: list-of-sources} - self.read_locations_of__class__ = {} - self.repr = None - self.extra_access_sets = {} - - if classdesc.basedesc: - self.basedef = classdesc.basedesc.getuniqueclassdef() - self.basedef.subdefs.append(self) - self.basedef.see_new_subclass(self) - else: - self.basedef = None - - self.parentdefs = dict.fromkeys(self.getmro()) - - def setup(self, sources): - # collect the (supposed constant) class attributes - for name, source in sources.items(): - self.add_source_for_attribute(name, source) - if self.bookkeeper: - self.bookkeeper.event('classdef_setup', self) - - def add_source_for_attribute(self, attr, source): - """Adds information about a constant source for an attribute. - """ - for cdef in self.getmro(): - if attr in cdef.attrs: - # the Attribute() exists already for this class (or a parent) - attrdef = cdef.attrs[attr] - s_prev_value = attrdef.s_value - attrdef.add_constant_source(self, source) - # we should reflow from all the reader's position, - # but as an optimization we try to see if the attribute - # has really been generalized - if attrdef.s_value != s_prev_value: - attrdef.mutated(cdef) # reflow from all read positions - return - else: - # remember the source in self.attr_sources - sources = self.attr_sources.setdefault(attr, []) - sources.append(source) - # register the source in any Attribute found in subclasses, - # to restore invariant (III) - # NB. add_constant_source() may discover new subdefs but the - # right thing will happen to them because self.attr_sources - # was already updated - if not source.instance_level: - for subdef in self.getallsubdefs(): - if attr in subdef.attrs: - attrdef = subdef.attrs[attr] - s_prev_value = attrdef.s_value - attrdef.add_constant_source(self, source) - if attrdef.s_value != s_prev_value: - attrdef.mutated(subdef) # reflow from all read positions - - def locate_attribute(self, attr): - while True: - for cdef in self.getmro(): - if attr in cdef.attrs: - return cdef - self.generalize_attr(attr) - # the return value will likely be 'self' now, but not always -- see - # test_annrpython.test_attr_moving_from_subclass_to_class_to_parent - - def find_attribute(self, attr): - return self.locate_attribute(attr).attrs[attr] - - def __repr__(self): - return "<ClassDef '%s'>" % (self.name,) - - def has_no_attrs(self): - for clsdef in self.getmro(): - if clsdef.attrs: - return False - return True - - def commonbase(self, other): - while other is not None and not self.issubclass(other): - other = other.basedef - return other - - def getmro(self): - while self is not None: - yield self - self = self.basedef - - def issubclass(self, otherclsdef): - return otherclsdef in self.parentdefs - - def getallsubdefs(self): - pending = [self] - seen = {} - for clsdef in pending: - yield clsdef - for sub in clsdef.subdefs: - if sub not in seen: - pending.append(sub) - seen[sub] = True - - def _generalize_attr(self, attr, s_value): - # first remove the attribute from subclasses -- including us! - # invariant (I) - subclass_attrs = [] - constant_sources = [] # [(classdef-of-origin, source)] - for subdef in self.getallsubdefs(): - if attr in subdef.attrs: - subclass_attrs.append(subdef.attrs[attr]) - del subdef.attrs[attr] - if attr in subdef.attr_sources: - # accumulate attr_sources for this attribute from all subclasses - lst = subdef.attr_sources[attr] - for source in lst: - constant_sources.append((subdef, source)) - del lst[:] # invariant (II) - - # accumulate attr_sources for this attribute from all parents, too - # invariant (III) - for superdef in self.getmro(): - if attr in superdef.attr_sources: - for source in superdef.attr_sources[attr]: - if not source.instance_level: - constant_sources.append((superdef, source)) - - # create the Attribute and do the generalization asked for - newattr = Attribute(attr, self.bookkeeper) - if s_value: - #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False): - # import pdb; pdb.set_trace() - newattr.s_value = s_value - - # keep all subattributes' values - for subattr in subclass_attrs: - newattr.merge(subattr, classdef=self) - - # store this new Attribute, generalizing the previous ones from - # subclasses -- invariant (A) - self.attrs[attr] = newattr - - # add the values of the pending constant attributes - # completes invariants (II) and (III) - for origin_classdef, source in constant_sources: - newattr.add_constant_source(origin_classdef, source) - - # reflow from all read positions - newattr.mutated(self) - - def generalize_attr(self, attr, s_value=None): - # if the attribute exists in a superclass, generalize there, - # as imposed by invariant (I) - for clsdef in self.getmro(): - if attr in clsdef.attrs: - clsdef._generalize_attr(attr, s_value) - break - else: - self._generalize_attr(attr, s_value) - - def about_attribute(self, name): - """This is the interface for the code generators to ask about - the annotation given to a attribute.""" - for cdef in self.getmro(): - if name in cdef.attrs: - s_result = cdef.attrs[name].s_value - if s_result != s_ImpossibleValue: - return s_result - else: - return None - return None - - def lookup_filter(self, pbc, name=None, flags={}): - """Selects the methods in the pbc that could possibly be seen by - a lookup performed on an instance of 'self', removing the ones - that cannot appear. - """ - d = [] - uplookup = None - updesc = None - for desc in pbc.descriptions: - # pick methods but ignore already-bound methods, which can come - # from an instance attribute - if (isinstance(desc, description.MethodDesc) - and desc.selfclassdef is None): - methclassdef = desc.originclassdef - if methclassdef is not self and methclassdef.issubclass(self): - pass # subclasses methods are always candidates - elif self.issubclass(methclassdef): - # upward consider only the best match - if uplookup is None or methclassdef.issubclass(uplookup): - uplookup = methclassdef - updesc = desc - continue - # for clsdef1 >= clsdef2, we guarantee that - # clsdef1.lookup_filter(pbc) includes - # clsdef2.lookup_filter(pbc) (see formal proof...) - else: - continue # not matching - # bind the method by giving it a selfclassdef. Use the - # more precise subclass that it's coming from. - desc = desc.bind_self(methclassdef, flags) - d.append(desc) - if uplookup is not None: - d.append(updesc.bind_self(self, flags)) - - if d: - return SomePBC(d, can_be_None=pbc.can_be_None) - elif pbc.can_be_None: - return s_None - else: - return s_ImpossibleValue - - def check_missing_attribute_update(self, name): - # haaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaack - # sometimes, new methods can show up on classes, added - # e.g. by W_TypeObject._freeze_() -- the multimethod - # implementations. Check that here... - found = False - parents = list(self.getmro()) - parents.reverse() - for base in parents: - if base.check_attr_here(name): - found = True - return found - - def check_attr_here(self, name): - source = self.classdesc.find_source_for(name) - if source is not None: - # oups! new attribute showed up - self.add_source_for_attribute(name, source) - # maybe it also showed up in some subclass? - for subdef in self.getallsubdefs(): - if subdef is not self: - subdef.check_attr_here(name) - return True - else: - return False - - def see_new_subclass(self, classdef): - for position in self.read_locations_of__class__: - self.bookkeeper.annotator.reflowfromposition(position) - if self.basedef is not None: - self.basedef.see_new_subclass(classdef) - - def read_attr__class__(self): - position = self.bookkeeper.position_key - self.read_locations_of__class__[position] = True - return SomePBC([subdef.classdesc for subdef in self.getallsubdefs()]) - - def _freeze_(self): - raise Exception("ClassDefs are used as knowntype for instances but cannot be used as immutablevalue arguments directly") - -# ____________________________________________________________ - -class InstanceSource(object): - instance_level = True - - def __init__(self, bookkeeper, obj): - self.bookkeeper = bookkeeper - self.obj = obj - - def s_get_value(self, classdef, name): - try: - v = getattr(self.obj, name) - except AttributeError: - all_enforced_attrs = classdef.classdesc.all_enforced_attrs - if all_enforced_attrs and name in all_enforced_attrs: - return s_ImpossibleValue - raise - s_value = self.bookkeeper.immutablevalue(v) - return s_value - - def all_instance_attributes(self): - result = getattr(self.obj, '__dict__', {}).keys() - tp = self.obj.__class__ - if isinstance(tp, type): - for basetype in tp.__mro__: - slots = basetype.__dict__.get('__slots__') - if slots: - if isinstance(slots, str): - result.append(slots) - else: - result.extend(slots) - return result - -class NoSuchAttrError(AnnotatorError): - """Raised when an attribute is found on a class where __slots__ - or _attrs_ forbits it.""" diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py new file mode 100644 --- /dev/null +++ b/rpython/annotator/classdesc.py @@ -0,0 +1,944 @@ +""" +Type inference for user-defined classes. +""" +from __future__ import absolute_import +import types + +from rpython.flowspace.model import Constant +from rpython.tool.flattenrec import FlattenRecursion +from rpython.tool.sourcetools import func_with_new_name +from rpython.tool.uid import Hashable +from rpython.annotator.model import ( + SomePBC, s_ImpossibleValue, unionof, s_None, AnnotatorError, SomeInteger, + SomeString, SomeImpossibleValue, SomeList, HarmlesslyBlocked) +from rpython.annotator.description import ( + Desc, FunctionDesc, MethodDesc, NODEFAULT) + + +# The main purpose of a ClassDef is to collect information about class/instance +# attributes as they are really used. An Attribute object is stored in the +# most general ClassDef where an attribute of that name is read/written: +# classdef.attrs = {'attrname': Attribute()} +# +# The following invariants hold: +# +# (A) if an attribute is read/written on an instance of class A, then the +# classdef of A or a parent class of A has an Attribute object corresponding +# to that name. +# +# (I) if B is a subclass of A, then they don't both have an Attribute for the +# same name. (All information from B's Attribute must be merged into A's.) +# +# Additionally, each ClassDef records an 'attr_sources': it maps attribute names +# to a list of 'source' objects that want to provide a constant value for this +# attribute at the level of this class. The attr_sources provide information +# higher in the class hierarchy than concrete Attribute()s. It is for the case +# where (so far or definitely) the user program only reads/writes the attribute +# at the level of a subclass, but a value for this attribute could possibly +# exist in the parent class or in an instance of a parent class. +# +# The point of not automatically forcing the Attribute instance up to the +# parent class which has a class attribute of the same name is apparent with +# multiple subclasses: +# +# A +# attr=s1 +# / \ +# / \ +# B C +# attr=s2 attr=s3 +# +# XXX this does not seem to be correct, but I don't know how to phrase +# it correctly. See test_specific_attributes in test_annrpython +# +# In this case, as long as 'attr' is only read/written from B or C, the +# Attribute on B says that it can be 's1 or s2', and the Attribute on C says +# it can be 's1 or s3'. Merging them into a single Attribute on A would give +# the more imprecise 's1 or s2 or s3'. +# +# The following invariant holds: +# +# (II) if a class A has an Attribute, the 'attr_sources' for the same name is +# empty. It is also empty on all subclasses of A. (The information goes +# into the Attribute directly in this case.) +# +# The following invariant holds: +# +# (III) for a class A, each attrsource that comes from the class (as opposed to +# from a prebuilt instance) must be merged into all Attributes of the +# same name in all subclasses of A, if any. (Parent class attributes can +# be visible in reads from instances of subclasses.) + +class Attribute(object): + # readonly-ness + # SomeThing-ness + # NB. an attribute is readonly if it is a constant class attribute. + # Both writing to the instance attribute and discovering prebuilt + # instances that have the attribute set will turn off readonly-ness. + + def __init__(self, name): + assert name != '__class__' + self.name = name + self.s_value = s_ImpossibleValue + self.readonly = True + self.attr_allowed = True + self.read_locations = set() + + def add_constant_source(self, classdef, source): + s_value = source.s_get_value(classdef, self.name) + if source.instance_level: + # a prebuilt instance source forces readonly=False, see above + self.modified(classdef) + s_new_value = unionof(self.s_value, s_value) + self.s_value = s_new_value + + def merge(self, other, classdef): + assert self.name == other.name + s_new_value = unionof(self.s_value, other.s_value) + self.s_value = s_new_value + if not other.readonly: + self.modified(classdef) + self.read_locations.update(other.read_locations) + + def validate(self, homedef): + s_newvalue = self.s_value + # check for after-the-fact method additions + if isinstance(s_newvalue, SomePBC): + attr = self.name + if s_newvalue.getKind() == MethodDesc: + # is method + if homedef.classdesc.read_attribute(attr, None) is None: + homedef.check_missing_attribute_update(attr) + + # check for attributes forbidden by slots or _attrs_ + if homedef.classdesc.all_enforced_attrs is not None: + if self.name not in homedef.classdesc.all_enforced_attrs: + self.attr_allowed = False + if not self.readonly: + raise NoSuchAttrError( + "the attribute %r goes here to %r, but it is " + "forbidden here" % (self.name, homedef)) + + def modified(self, classdef='?'): + self.readonly = False + if not self.attr_allowed: + from rpython.annotator.bookkeeper import getbookkeeper + bk = getbookkeeper() + classdesc = classdef.classdesc + locations = bk.getattr_locations(classdesc, self.name) + raise NoSuchAttrError( + "Attribute %r on %r should be read-only.\n" % (self.name, + classdef) + + "This error can be caused by another 'getattr' that promoted\n" + "the attribute here; the list of read locations is:\n" + + '\n'.join([str(loc[0]) for loc in locations])) + +class ClassDef(object): + "Wraps a user class." + + def __init__(self, bookkeeper, classdesc): + self.bookkeeper = bookkeeper + self.attrs = {} # {name: Attribute} + self.classdesc = classdesc + self.name = self.classdesc.name + self.shortname = self.name.split('.')[-1] + self.subdefs = [] + self.attr_sources = {} # {name: list-of-sources} + self.read_locations_of__class__ = {} + self.repr = None + self.extra_access_sets = {} + self.instances_seen = set() + + if classdesc.basedesc: + self.basedef = classdesc.basedesc.getuniqueclassdef() + self.basedef.subdefs.append(self) + self.basedef.see_new_subclass(self) + else: + self.basedef = None + + self.parentdefs = dict.fromkeys(self.getmro()) + + def setup(self, sources): + # collect the (supposed constant) class attributes + for name, source in sources.items(): + self.add_source_for_attribute(name, source) + if self.bookkeeper: + self.bookkeeper.event('classdef_setup', self) + + def s_getattr(self, attrname, flags): + attrdef = self.find_attribute(attrname) + s_result = attrdef.s_value + # hack: if s_result is a set of methods, discard the ones + # that can't possibly apply to an instance of self. + # XXX do it more nicely + if isinstance(s_result, SomePBC): + s_result = self.lookup_filter(s_result, attrname, flags) + elif isinstance(s_result, SomeImpossibleValue): + self.check_missing_attribute_update(attrname) + # blocking is harmless if the attribute is explicitly listed + # in the class or a parent class. + for basedef in self.getmro(): + if basedef.classdesc.all_enforced_attrs is not None: + if attrname in basedef.classdesc.all_enforced_attrs: + raise HarmlesslyBlocked("get enforced attr") + elif isinstance(s_result, SomeList): + s_result = self.classdesc.maybe_return_immutable_list( + attrname, s_result) + return s_result + + def add_source_for_attribute(self, attr, source): + """Adds information about a constant source for an attribute. + """ + for cdef in self.getmro(): + if attr in cdef.attrs: + # the Attribute() exists already for this class (or a parent) + attrdef = cdef.attrs[attr] + s_prev_value = attrdef.s_value + attrdef.add_constant_source(self, source) + # we should reflow from all the reader's position, + # but as an optimization we try to see if the attribute + # has really been generalized + if attrdef.s_value != s_prev_value: + self.bookkeeper.update_attr(cdef, attrdef) + return + else: + # remember the source in self.attr_sources + sources = self.attr_sources.setdefault(attr, []) + sources.append(source) + # register the source in any Attribute found in subclasses, + # to restore invariant (III) + # NB. add_constant_source() may discover new subdefs but the + # right thing will happen to them because self.attr_sources + # was already updated + if not source.instance_level: + for subdef in self.getallsubdefs(): + if attr in subdef.attrs: + attrdef = subdef.attrs[attr] + s_prev_value = attrdef.s_value + attrdef.add_constant_source(self, source) + if attrdef.s_value != s_prev_value: + self.bookkeeper.update_attr(subdef, attrdef) + + def get_owner(self, attrname): + """Return the classdef owning the attribute `attrname`.""" + for cdef in self.getmro(): + if attrname in cdef.attrs: + return cdef + else: + return None + + + def locate_attribute(self, attr): + cdef = self.get_owner(attr) + if cdef: + return cdef + else: + self._generalize_attr(attr, s_value=None) + return self + + def find_attribute(self, attr): + return self.locate_attribute(attr).attrs[attr] + + def __repr__(self): + return "<ClassDef '%s'>" % (self.name,) + + def has_no_attrs(self): + for clsdef in self.getmro(): + if clsdef.attrs: + return False + return True + + def commonbase(self, other): + while other is not None and not self.issubclass(other): + other = other.basedef + return other + + def getmro(self): + while self is not None: + yield self + self = self.basedef + + def issubclass(self, otherclsdef): + return otherclsdef in self.parentdefs + + def getallsubdefs(self): + pending = [self] + seen = {} + for clsdef in pending: + yield clsdef + for sub in clsdef.subdefs: + if sub not in seen: + pending.append(sub) + seen[sub] = True + + def _generalize_attr(self, attr, s_value): + # create the Attribute and do the generalization asked for + newattr = Attribute(attr) + if s_value: + newattr.s_value = s_value + + # remove the attribute from subclasses -- including us! + # invariant (I) + constant_sources = [] # [(classdef-of-origin, source)] + for subdef in self.getallsubdefs(): + if attr in subdef.attrs: + subattr = subdef.attrs[attr] + newattr.merge(subattr, classdef=self) + del subdef.attrs[attr] + if attr in subdef.attr_sources: + # accumulate attr_sources for this attribute from all subclasses + lst = subdef.attr_sources[attr] + for source in lst: + constant_sources.append((subdef, source)) + del lst[:] # invariant (II) + + # accumulate attr_sources for this attribute from all parents, too + # invariant (III) + for superdef in self.getmro(): + if attr in superdef.attr_sources: + for source in superdef.attr_sources[attr]: + if not source.instance_level: + constant_sources.append((superdef, source)) + + # store this new Attribute, generalizing the previous ones from + # subclasses -- invariant (A) + self.attrs[attr] = newattr + + # add the values of the pending constant attributes + # completes invariants (II) and (III) + for origin_classdef, source in constant_sources: + newattr.add_constant_source(origin_classdef, source) + + # reflow from all read positions + self.bookkeeper.update_attr(self, newattr) + + def generalize_attr(self, attr, s_value=None): + # if the attribute exists in a superclass, generalize there, + # as imposed by invariant (I) + clsdef = self.get_owner(attr) + if clsdef: + clsdef._generalize_attr(attr, s_value) + else: + self._generalize_attr(attr, s_value) + + def about_attribute(self, name): + """This is the interface for the code generators to ask about + the annotation given to a attribute.""" + for cdef in self.getmro(): + if name in cdef.attrs: + s_result = cdef.attrs[name].s_value + if s_result != s_ImpossibleValue: + return s_result + else: + return None + return None + + def lookup_filter(self, pbc, name=None, flags={}): + """Selects the methods in the pbc that could possibly be seen by + a lookup performed on an instance of 'self', removing the ones + that cannot appear. + """ + d = [] + uplookup = None + updesc = None + for desc in pbc.descriptions: + # pick methods but ignore already-bound methods, which can come + # from an instance attribute + if (isinstance(desc, MethodDesc) and desc.selfclassdef is None): + methclassdef = desc.originclassdef + if methclassdef is not self and methclassdef.issubclass(self): + pass # subclasses methods are always candidates + elif self.issubclass(methclassdef): + # upward consider only the best match + if uplookup is None or methclassdef.issubclass(uplookup): + uplookup = methclassdef + updesc = desc + continue + # for clsdef1 >= clsdef2, we guarantee that + # clsdef1.lookup_filter(pbc) includes + # clsdef2.lookup_filter(pbc) (see formal proof...) + else: + continue # not matching + # bind the method by giving it a selfclassdef. Use the + # more precise subclass that it's coming from. + desc = desc.bind_self(methclassdef, flags) + d.append(desc) + if uplookup is not None: + d.append(updesc.bind_self(self, flags)) + + if d: + return SomePBC(d, can_be_None=pbc.can_be_None) + elif pbc.can_be_None: + return s_None + else: + return s_ImpossibleValue + + def check_missing_attribute_update(self, name): + # haaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaack + # sometimes, new methods can show up on classes, added + # e.g. by W_TypeObject._freeze_() -- the multimethod + # implementations. Check that here... + found = False + parents = list(self.getmro()) + parents.reverse() + for base in parents: + if base.check_attr_here(name): + found = True + return found + + def check_attr_here(self, name): + source = self.classdesc.find_source_for(name) + if source is not None: + # oups! new attribute showed up + self.add_source_for_attribute(name, source) + # maybe it also showed up in some subclass? + for subdef in self.getallsubdefs(): + if subdef is not self: + subdef.check_attr_here(name) + return True + else: + return False + + _see_instance_flattenrec = FlattenRecursion() + + def see_instance(self, x): + assert isinstance(x, self.classdesc.pyobj) + key = Hashable(x) + if key in self.instances_seen: + return + self.instances_seen.add(key) + self.bookkeeper.event('mutable', x) + source = InstanceSource(self.bookkeeper, x) + def delayed(): + for attr in source.all_instance_attributes(): + self.add_source_for_attribute(attr, source) + # ^^^ can trigger reflowing + self._see_instance_flattenrec(delayed) + + def see_new_subclass(self, classdef): + for position in self.read_locations_of__class__: + self.bookkeeper.annotator.reflowfromposition(position) + if self.basedef is not None: + self.basedef.see_new_subclass(classdef) + + def read_attr__class__(self): + position = self.bookkeeper.position_key + self.read_locations_of__class__[position] = True + return SomePBC([subdef.classdesc for subdef in self.getallsubdefs()]) + + def _freeze_(self): + raise Exception("ClassDefs are used as knowntype for instances but " + "cannot be used as immutablevalue arguments directly") + +# ____________________________________________________________ + +class InstanceSource(object): + instance_level = True + + def __init__(self, bookkeeper, obj): + self.bookkeeper = bookkeeper + self.obj = obj + + def s_get_value(self, classdef, name): + try: + v = getattr(self.obj, name) + except AttributeError: + all_enforced_attrs = classdef.classdesc.all_enforced_attrs + if all_enforced_attrs and name in all_enforced_attrs: + return s_ImpossibleValue + raise + s_value = self.bookkeeper.immutablevalue(v) + return s_value + + def all_instance_attributes(self): + result = getattr(self.obj, '__dict__', {}).keys() + tp = self.obj.__class__ + if isinstance(tp, type): + for basetype in tp.__mro__: + slots = basetype.__dict__.get('__slots__') + if slots: + if isinstance(slots, str): + result.append(slots) + else: + result.extend(slots) + return result + +class NoSuchAttrError(AnnotatorError): + """Raised when an attribute is found on a class where __slots__ + or _attrs_ forbits it.""" + + +def is_mixin(cls): + return cls.__dict__.get('_mixin_', False) + + +class ClassDesc(Desc): + knowntype = type + instance_level = False + all_enforced_attrs = None # or a set + _detect_invalid_attrs = None + + def __init__(self, bookkeeper, cls, + name=None, basedesc=None, classdict=None): + super(ClassDesc, self).__init__(bookkeeper, cls) + if '__NOT_RPYTHON__' in cls.__dict__: + raise AnnotatorError('Bad class') + + if name is None: + name = cls.__module__ + '.' + cls.__name__ + self.name = name + self.basedesc = basedesc + if classdict is None: + classdict = {} # populated below + self.classdict = classdict # {attr: Constant-or-Desc} + if cls.__dict__.get('_annspecialcase_', ''): + raise AnnotatorError( + "Class specialization has been removed. The " + "'_annspecialcase_' class tag is now unsupported.") + self.classdef = None + + if is_mixin(cls): + raise AnnotatorError("cannot use directly the class %r because " + "it is a _mixin_" % (cls,)) + + assert cls.__module__ != '__builtin__' + baselist = list(cls.__bases__) + + # special case: skip BaseException, and pretend + # that all exceptions ultimately inherit from Exception instead + # of BaseException (XXX hack) + if cls is Exception: + baselist = [] + elif baselist == [BaseException]: + baselist = [Exception] + + immutable_fields = cls.__dict__.get('_immutable_fields_', []) + # To prevent confusion, we forbid strings. Any other bona fide sequence + # of strings is OK. + if isinstance(immutable_fields, basestring): + raise AnnotatorError( + "In class %s, '_immutable_fields_' must be a sequence of " + "attribute names, not a string." % cls) + self.immutable_fields = set(immutable_fields) + + mixins_before = [] + mixins_after = [] + base = object + for b1 in baselist: + if b1 is object: + continue + if is_mixin(b1): + if base is object: + mixins_before.append(b1) + else: + mixins_after.append(b1) + else: + assert base is object, ("multiple inheritance only supported " + "with _mixin_: %r" % (cls,)) + base = b1 + if mixins_before and mixins_after: + raise Exception("unsupported: class %r has mixin bases both" + " before and after the regular base" % (self,)) + self.add_mixins(mixins_after, check_not_in=base) + self.add_mixins(mixins_before) + self.add_sources_for_class(cls) + + if base is not object: + self.basedesc = bookkeeper.getdesc(base) + + if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__: + attrs = {} + for decl in ('__slots__', '_attrs_'): + decl = cls.__dict__.get(decl, []) + if isinstance(decl, str): + decl = (decl,) + decl = dict.fromkeys(decl) + attrs.update(decl) + if self.basedesc is not None: + if self.basedesc.all_enforced_attrs is None: + raise Exception("%r has slots or _attrs_, " + "but not its base class" % (cls,)) + attrs.update(self.basedesc.all_enforced_attrs) + self.all_enforced_attrs = attrs + + if (self.is_builtin_exception_class() and + self.all_enforced_attrs is None): + if cls not in FORCE_ATTRIBUTES_INTO_CLASSES: + self.all_enforced_attrs = [] # no attribute allowed + + def add_source_attribute(self, name, value, mixin=False): + if isinstance(value, property): + # special case for property object + if value.fget is not None: + newname = name + '__getter__' + func = func_with_new_name(value.fget, newname) + self.add_source_attribute(newname, func, mixin) + if value.fset is not None: + newname = name + '__setter__' + func = func_with_new_name(value.fset, newname) + self.add_source_attribute(newname, func, mixin) + self.classdict[name] = Constant(value) + return + + if isinstance(value, types.FunctionType): + # for debugging + if not hasattr(value, 'class_'): + value.class_ = self.pyobj + if mixin: + # make a new copy of the FunctionDesc for this class, + # but don't specialize further for all subclasses + funcdesc = FunctionDesc(self.bookkeeper, value) + self.classdict[name] = funcdesc + return + # NB. if value is, say, AssertionError.__init__, then we + # should not use getdesc() on it. Never. The problem is + # that the py lib has its own AssertionError.__init__ which + # is of type FunctionType. But bookkeeper.immutablevalue() + # will do the right thing in s_get_value(). + if isinstance(value, staticmethod) and mixin: + # make a new copy of staticmethod + func = value.__get__(42) + value = staticmethod(func_with_new_name(func, func.__name__)) + + if type(value) in MemberDescriptorTypes: + # skip __slots__, showing up in the class as 'member' objects + return + if name == '__init__' and self.is_builtin_exception_class(): + # pretend that built-in exceptions have no __init__, + # unless explicitly specified in builtin.py + from rpython.annotator.builtin import BUILTIN_ANALYZERS + value = getattr(value, 'im_func', value) + if value not in BUILTIN_ANALYZERS: + return + self.classdict[name] = Constant(value) + + def add_mixins(self, mixins, check_not_in=object): + if not mixins: + return + A = type('tmp', tuple(mixins) + (object,), {}) + mro = A.__mro__ + assert mro[0] is A and mro[-1] is object + mro = mro[1:-1] + # + skip = set() + def add(cls): + if cls is not object: + for base in cls.__bases__: + add(base) + for name in cls.__dict__: + skip.add(name) + add(check_not_in) + # + for base in reversed(mro): + assert is_mixin(base), ( + "Mixin class %r has non mixin base class %r" % (mixins, base)) + for name, value in base.__dict__.items(): + if name in skip: + continue + self.add_source_attribute(name, value, mixin=True) + if '_immutable_fields_' in base.__dict__: + self.immutable_fields.update( + set(base.__dict__['_immutable_fields_'])) + + + def add_sources_for_class(self, cls): + for name, value in cls.__dict__.items(): + self.add_source_attribute(name, value) + + def getclassdef(self, key): + return self.getuniqueclassdef() + + def _init_classdef(self): + classdef = ClassDef(self.bookkeeper, self) + self.bookkeeper.classdefs.append(classdef) + self.classdef = classdef + + # forced attributes + cls = self.pyobj + if cls in FORCE_ATTRIBUTES_INTO_CLASSES: + for name, s_value in FORCE_ATTRIBUTES_INTO_CLASSES[cls].items(): + classdef.generalize_attr(name, s_value) + classdef.find_attribute(name).modified(classdef) + + # register all class attributes as coming from this ClassDesc + # (as opposed to prebuilt instances) + classsources = {} + for attr in self.classdict: + classsources[attr] = self # comes from this ClassDesc + classdef.setup(classsources) + # look for a __del__ method and annotate it if it's there + if '__del__' in self.classdict: + from rpython.annotator.model import s_None, SomeInstance + s_func = self.s_read_attribute('__del__') + args_s = [SomeInstance(classdef)] + s = self.bookkeeper.emulate_pbc_call(classdef, s_func, args_s) + assert s_None.contains(s) + return classdef + + def getuniqueclassdef(self): + if self.classdef is None: + self._init_classdef() + return self.classdef + + def pycall(self, whence, args, s_previous_result, op=None): + from rpython.annotator.model import SomeInstance, SomeImpossibleValue + classdef = self.getuniqueclassdef() + s_instance = SomeInstance(classdef) + # look up __init__ directly on the class, bypassing the normal + # lookup mechanisms ClassDef (to avoid influencing Attribute placement) + s_init = self.s_read_attribute('__init__') + if isinstance(s_init, SomeImpossibleValue): + # no __init__: check that there are no constructor args + if not self.is_exception_class(): + try: + args.fixedunpack(0) + except ValueError: + raise Exception("default __init__ takes no argument" + " (class %s)" % (self.name,)) + elif self.pyobj is Exception: + # check explicitly against "raise Exception, x" where x + # is a low-level exception pointer + try: + [s_arg] = args.fixedunpack(1) + except ValueError: + pass + else: + from rpython.rtyper.llannotation import SomePtr + assert not isinstance(s_arg, SomePtr) + else: + # call the constructor + args = args.prepend(s_instance) + s_init.call(args) + return s_instance + + def is_exception_class(self): + return issubclass(self.pyobj, BaseException) + + def is_builtin_exception_class(self): + if self.is_exception_class(): + if self.pyobj.__module__ == 'exceptions': + return True + if issubclass(self.pyobj, AssertionError): + return True + return False + + def lookup(self, name): + cdesc = self + while name not in cdesc.classdict: + cdesc = cdesc.basedesc + if cdesc is None: + return None + else: + return cdesc + + def get_param(self, name, default=None, inherit=True): + cls = self.pyobj + if inherit: + return getattr(cls, name, default) + else: + return cls.__dict__.get(name, default) + + def read_attribute(self, name, default=NODEFAULT): + cdesc = self.lookup(name) + if cdesc is None: + if default is NODEFAULT: + raise AttributeError + else: + return default + else: + return cdesc.classdict[name] + + def s_read_attribute(self, name): + # look up an attribute in the class + cdesc = self.lookup(name) + if cdesc is None: + return s_ImpossibleValue + else: + # delegate to s_get_value to turn it into an annotation + return cdesc.s_get_value(None, name) + + def s_get_value(self, classdef, name): + obj = self.classdict[name] + if isinstance(obj, Constant): + value = obj.value + if isinstance(value, staticmethod): # special case + value = value.__get__(42) + classdef = None # don't bind + elif isinstance(value, classmethod): + raise AnnotatorError("classmethods are not supported") + s_value = self.bookkeeper.immutablevalue(value) + if classdef is not None: + s_value = s_value.bind_callables_under(classdef, name) + elif isinstance(obj, Desc): + if classdef is not None: + obj = obj.bind_under(classdef, name) + s_value = SomePBC([obj]) + else: + raise TypeError("classdict should not contain %r" % (obj,)) + return s_value + + def create_new_attribute(self, name, value): + assert name not in self.classdict, "name clash: %r" % (name,) + self.classdict[name] = Constant(value) + + def find_source_for(self, name): + if name in self.classdict: + return self + # check whether there is a new attribute + cls = self.pyobj + if name in cls.__dict__: + self.add_source_attribute(name, cls.__dict__[name]) + if name in self.classdict: + return self + return None + + def maybe_return_immutable_list(self, attr, s_result): + # hack: 'x.lst' where lst is listed in _immutable_fields_ as + # either 'lst[*]' or 'lst?[*]' + # should really return an immutable list as a result. Implemented + # by changing the result's annotation (but not, of course, doing an + # actual copy in the rtyper). Tested in rpython.rtyper.test.test_rlist, + # test_immutable_list_out_of_instance. + if self._detect_invalid_attrs and attr in self._detect_invalid_attrs: + raise Exception("field %r was migrated to %r from a subclass in " + "which it was declared as _immutable_fields_" % + (attr, self.pyobj)) + search1 = '%s[*]' % (attr,) + search2 = '%s?[*]' % (attr,) + cdesc = self + while cdesc is not None: + immutable_fields = cdesc.immutable_fields + if immutable_fields: + if (search1 in immutable_fields or search2 in immutable_fields): + s_result.listdef.never_resize() + s_copy = s_result.listdef.offspring() + s_copy.listdef.mark_as_immutable() + # + cdesc = cdesc.basedesc + while cdesc is not None: + if cdesc._detect_invalid_attrs is None: + cdesc._detect_invalid_attrs = set() + cdesc._detect_invalid_attrs.add(attr) + cdesc = cdesc.basedesc + # + return s_copy + cdesc = cdesc.basedesc + return s_result # common case + + @staticmethod + def consider_call_site(descs, args, s_result, op): + descs[0].getcallfamily() + descs[0].mergecallfamilies(*descs[1:]) + from rpython.annotator.model import SomeInstance, SomePBC, s_None + if len(descs) == 1: + # call to a single class, look at the result annotation + # in case it was specialized + if not isinstance(s_result, SomeInstance): + raise Exception("calling a class didn't return an instance??") + classdefs = [s_result.classdef] + else: + # call to multiple classes: specialization not supported + classdefs = [desc.getuniqueclassdef() for desc in descs] + # If some of the classes have an __init__ and others not, then + # we complain, even though in theory it could work if all the + # __init__s take no argument. But it's messy to implement, so + # let's just say it is not RPython and you have to add an empty + # __init__ to your base class. + has_init = False + for desc in descs: + s_init = desc.s_read_attribute('__init__') + has_init |= isinstance(s_init, SomePBC) + basedesc = ClassDesc.getcommonbase(descs) + s_init = basedesc.s_read_attribute('__init__') + parent_has_init = isinstance(s_init, SomePBC) + if has_init and not parent_has_init: + raise AnnotatorError( + "some subclasses among %r declare __init__()," + " but not the common parent class" % (descs,)) + # make a PBC of MethodDescs, one for the __init__ of each class + initdescs = [] + for desc, classdef in zip(descs, classdefs): + s_init = desc.s_read_attribute('__init__') + if isinstance(s_init, SomePBC): + assert len(s_init.descriptions) == 1, ( + "unexpected dynamic __init__?") + initfuncdesc, = s_init.descriptions + if isinstance(initfuncdesc, FunctionDesc): + from rpython.annotator.bookkeeper import getbookkeeper + initmethdesc = getbookkeeper().getmethoddesc( + initfuncdesc, classdef, classdef, '__init__') + initdescs.append(initmethdesc) + # register a call to exactly these __init__ methods + if initdescs: + initdescs[0].mergecallfamilies(*initdescs[1:]) + MethodDesc.consider_call_site(initdescs, args, s_None, op) + + def getallbases(self): + desc = self + while desc is not None: + yield desc + desc = desc.basedesc + + @staticmethod + def getcommonbase(descs): + commondesc = descs[0] + for desc in descs[1:]: + allbases = set(commondesc.getallbases()) + while desc not in allbases: + assert desc is not None, "no common base for %r" % (descs,) + desc = desc.basedesc + commondesc = desc + return commondesc + + def rowkey(self): + return self + + def getattrfamily(self, attrname): + "Get the ClassAttrFamily object for attrname. Possibly creates one." + access_sets = self.bookkeeper.get_classpbc_attr_families(attrname) + _, _, attrfamily = access_sets.find(self) + return attrfamily + + def queryattrfamily(self, attrname): + """Retrieve the ClassAttrFamily object for attrname if there is one, + otherwise return None.""" + access_sets = self.bookkeeper.get_classpbc_attr_families(attrname) + try: + return access_sets[self] + except KeyError: + return None + _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
