Author: Remi Meier <remi.me...@gmail.com> Branch: stmgc-c8 Changeset: r82484:4d0277154891 Date: 2016-02-24 17:34 +0100 http://bitbucket.org/pypy/pypy/changeset/4d0277154891/
Log: merge and fix tons of things diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -123,3 +123,13 @@ .. branch: fix-cpython-ssl-tests-2.7 Fix SSL tests by importing cpython's patch + +.. branch: remove-getfield-pure + +Remove pure variants of ``getfield_gc_*`` operations from the JIT. Relevant +optimizations instead consult the field descriptor to determine the purity of +the operation. Additionally, pure ``getfield`` operations are now handled +entirely by `rpython/jit/metainterp/optimizeopt/heap.py` rather than +`rpython/jit/metainterp/optimizeopt/pure.py`, which can result in better codegen +for traces containing a large number of pure getfield operations. + diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py --- a/pypy/module/cpyext/dictobject.py +++ b/pypy/module/cpyext/dictobject.py @@ -59,7 +59,7 @@ return None return borrow_from(w_dict, w_res) -@cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) +@cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) def PyDict_DelItemString(space, w_dict, key_ptr): """Remove the entry in dictionary p which has a key specified by the string key. Return 0 on success or -1 on failure.""" diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -128,7 +128,7 @@ filename = "<string>" return run_string(space, source, filename, start, w_globals, w_locals) -@cpython_api([rffi.CCHARP, rffi.INT_real, PyObject, PyObject, +@cpython_api([CONST_STRING, rffi.INT_real, PyObject, PyObject, PyCompilerFlagsPtr], PyObject) def PyRun_StringFlags(space, source, start, w_globals, w_locals, flagsptr): """Execute Python source code from str in the context specified by the @@ -189,7 +189,7 @@ pi[0] = space.getindex_w(w_obj, None) return 1 -@cpython_api([rffi.CCHARP, rffi.CCHARP, rffi.INT_real, PyCompilerFlagsPtr], +@cpython_api([CONST_STRING, CONST_STRING, rffi.INT_real, PyCompilerFlagsPtr], PyObject) def Py_CompileStringFlags(space, source, filename, start, flagsptr): """Parse and compile the Python source code in str, returning the diff --git a/pypy/module/cpyext/pystrtod.py b/pypy/module/cpyext/pystrtod.py --- a/pypy/module/cpyext/pystrtod.py +++ b/pypy/module/cpyext/pystrtod.py @@ -1,6 +1,6 @@ import errno from pypy.interpreter.error import OperationError -from pypy.module.cpyext.api import cpython_api +from pypy.module.cpyext.api import cpython_api, CONST_STRING from pypy.module.cpyext.pyobject import PyObject from rpython.rlib import rdtoa from rpython.rlib import rfloat @@ -22,7 +22,7 @@ rfloat.DIST_NAN: Py_DTST_NAN } -@cpython_api([rffi.CCHARP, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) +@cpython_api([CONST_STRING, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) @jit.dont_look_inside # direct use of _get_errno() def PyOS_string_to_double(space, s, endptr, w_overflow_exception): """Convert a string s to a double, raising a Python diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -241,7 +241,6 @@ self.emit_gc_store_or_indexed(op, ptr_box, index_box, value_box, fieldsize, itemsize, ofs) elif op.getopnum() in (rop.GETFIELD_GC_I, rop.GETFIELD_GC_F, rop.GETFIELD_GC_R, - rop.GETFIELD_GC_PURE_I, rop.GETFIELD_GC_PURE_F, rop.GETFIELD_GC_PURE_R, rop.GETFIELD_RAW_I, rop.GETFIELD_RAW_F, rop.GETFIELD_RAW_R): ofs, itemsize, sign = unpack_fielddescr(op.getdescr()) ptr_box = op.getarg(0) diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -1,7 +1,8 @@ from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler from rpython.jit.backend.llsupport.descr import ( CallDescr, ArrayOrFieldDescr, unpack_fielddescr) -from rpython.jit.metainterp.resoperation import ResOperation, rop +from rpython.jit.metainterp.resoperation import ( + ResOperation, rop, ResOpWithDescr, OpHelpers) from rpython.jit.metainterp.history import ConstInt from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import (have_debug_prints, debug_start, debug_stop, @@ -129,12 +130,17 @@ newop = GcRewriterAssembler.emit_gc_load_or_indexed( self, op, ptr_box, index_box, itemsize, factor, offset, sign, type) ptr_box = newop.getarg(0) - if (op and not op.is_always_pure() and ptr_box.type == 'r' - and ptr_box not in self.read_barrier_applied - and not self.write_barrier_applied(ptr_box)): - op1 = ResOperation(rop.STM_READ, [ptr_box], None) - self.read_barrier_applied[ptr_box] = None - self.emit_op(op1) + if op: + is_pure = op.is_always_pure() + if not is_pure and isinstance(op, ResOpWithDescr): + is_pure = OpHelpers.is_pure_with_descr(op.getopnum(), op.getdescr()) + if (ptr_box.type == 'r' # not raw + and not is_pure # needs stm_read + and ptr_box not in self.read_barrier_applied + and not self.write_barrier_applied(ptr_box)): + op1 = ResOperation(rop.STM_READ, [ptr_box], None) + self.read_barrier_applied[ptr_box] = None + self.emit_op(op1) return newop diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -1338,7 +1338,7 @@ self.check_rewrite(""" [p1, p3, i1, p4] - p2 = getfield_gc%(pure)s_r(p1, descr=uxdescr) + p2 = getfield_gc_r(p1, descr=uxdescr) i4 = getarrayitem_gc%(pure)s_i(p4, i1, descr=vdescr) jump(p2) """ % d, """ diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1156,7 +1156,6 @@ 'ARRAYLEN_GC/1d/i', 'STRLEN/1/i', 'STRGETITEM/2/i', - 'GETFIELD_GC_PURE/1d/rfi', 'GETARRAYITEM_GC_PURE/2d/rfi', #'GETFIELD_RAW_PURE/1d/rfi', these two operations not useful and #'GETARRAYITEM_RAW_PURE/2d/fi', dangerous when unrolling speculatively diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -77,7 +77,7 @@ def gct_fv_gc_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size, c_offset_to_length): - # XXX same behavior for zero=True: in theory that's wrong + # XXX same behavior for zero=True: in theory that's wrong if c_offset_to_length is None: v_raw = hop.genop("direct_call", [self.malloc_varsize_no_length_ptr, v_length, @@ -159,6 +159,11 @@ resulttype = lltype.Signed) hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result) + def gcheader_initdata(self, defnode): + hdr = lltype.malloc(self.HDR, immortal=True) + hdr.hash = lltype.identityhash_nocache(defnode.obj._as_ptr()) + return hdr._obj + ########## weakrefs ########## # Boehm: weakref objects are small structures containing only a Boehm diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -427,7 +427,6 @@ s_gcref = SomePtr(llmemory.GCREF) gcdata = self.gcdata - translator = self.translator #use the GC flag to find which malloc method to use #malloc_zero_filled == Ture -> malloc_fixedsize/varsize_clear #malloc_zero_filled == Flase -> malloc_fixedsize/varsize @@ -1615,6 +1614,24 @@ resulttype=llmemory.Address) llops.genop('raw_memclear', [v_adr, v_totalsize]) + def gcheader_initdata(self, defnode): + o = lltype.top_container(defnode.obj) + needs_hash = self.get_prebuilt_hash(o) is not None + hdr = self.gc_header_for(o, needs_hash) + return hdr._obj + + def get_prebuilt_hash(self, obj): + # for prebuilt objects that need to have their hash stored and + # restored. Note that only structures that are StructNodes all + # the way have their hash stored (and not e.g. structs with var- + # sized arrays at the end). 'obj' must be the top_container. + TYPE = lltype.typeOf(obj) + if not isinstance(TYPE, lltype.GcStruct): + return None + if TYPE._is_varsize(): + return None + return getattr(obj, '_hash_cache_', None) + def emit_stm_memclearinit(self, llops, v_size, c_size, c_fixedofs, v_a): assert self.translator.config.translation.stm if c_size is None: diff --git a/rpython/memory/gctransform/refcounting.py b/rpython/memory/gctransform/refcounting.py --- a/rpython/memory/gctransform/refcounting.py +++ b/rpython/memory/gctransform/refcounting.py @@ -285,3 +285,7 @@ resulttype=llmemory.Address) hop.genop("direct_call", [self.identityhash_ptr, v_adr], resultvar=hop.spaceop.result) + + def gcheader_initdata(self, defnode): + top = lltype.top_container(defnode.obj) + return self.gcheaderbuilder.header_of_object(top)._obj diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -284,6 +284,14 @@ hop.genop("cast_opaque_ptr", [v_result], resultvar=op.result) hop.genop("stm_set_into_obj", [v_result, c_ofstolength, v_length]) + def get_prebuilt_hash(self, obj): + return None # done differently with the stmgc + + def get_stm_prebuilt_hash(self, obj): + h = BaseFrameworkGCTransformer.get_prebuilt_hash(self, obj) + if h is None: + h = lltype.identityhash(obj._as_ptr()) + return h diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py --- a/rpython/memory/gctransform/test/test_framework.py +++ b/rpython/memory/gctransform/test/test_framework.py @@ -48,7 +48,7 @@ cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph @@ -129,7 +129,7 @@ t.config.translation.gc = gc cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() def test_no_collect_stm(): test_no_collect("stmgc") @@ -161,7 +161,8 @@ t.config.translation.gc = gc cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=gcpolicy) - f = py.test.raises(Exception, cbuild.generate_graphs_for_llinterp) + with py.test.raises(Exception) as f: + cbuild.build_database() expected = "'no_collect' function can trigger collection: <function g at " assert str(f.value).startswith(expected) @@ -188,7 +189,8 @@ t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) - f = py.test.raises(Exception, cbuild.generate_graphs_for_llinterp) + with py.test.raises(Exception) as f: + cbuild.build_database() assert 'can cause the GC to be called' in str(f.value) assert 'trace_func' in str(f.value) assert 'MyStructure' in str(f.value) @@ -283,7 +285,7 @@ t.config.translation.gc = gc cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -323,7 +325,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -356,7 +358,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -389,7 +391,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -427,7 +429,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -455,7 +457,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -490,7 +492,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -525,7 +527,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -555,7 +557,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() @@ -667,7 +669,7 @@ t.config.translation.gc = "stmgc" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() ff = graphof(t, f) #ff.show() diff --git a/rpython/memory/gctransform/test/test_transform.py b/rpython/memory/gctransform/test/test_transform.py --- a/rpython/memory/gctransform/test/test_transform.py +++ b/rpython/memory/gctransform/test/test_transform.py @@ -17,7 +17,7 @@ t = rtype(f, [s_list_of_strings]) # XXX we shouldn't need an actual gcpolicy here. cbuild = CStandaloneBuilder(t, f, t.config, gcpolicy=self.gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + cbuild.build_database() graph = cbuild.getentrypointptr()._obj.graph llinterp = LLInterpreter(t.rtyper) if option.view: diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -113,21 +113,14 @@ self.seen_graphs.add(graph) self.minimal_transform.add(graph) - def prepare_inline_helpers(self, graphs): + def inline_helpers(self, graphs): from rpython.translator.backendopt.inline import iter_callsites + raise_analyzer = RaiseAnalyzer(self.translator) for graph in graphs: - self.graph_dependencies[graph] = {} + to_enum = [] for called, block, i in iter_callsites(graph, None): if called in self.graphs_to_inline: - self.graph_dependencies[graph][called] = True - self.prepared = True - - def inline_helpers(self, graph): - if not self.prepared: - raise Exception("Need to call prepare_inline_helpers first") - if self.inline: - raise_analyzer = RaiseAnalyzer(self.translator) - to_enum = self.graph_dependencies.get(graph, self.graphs_to_inline) + to_enum.append(called) must_constfold = False for inline_graph in to_enum: try: @@ -378,6 +371,10 @@ return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP), lltype.nullptr(ARRAY_TYPEID_MAP))) + def get_prebuilt_hash(self, obj): + return None + + class MinimalGCTransformer(BaseGCTransformer): def __init__(self, parenttransformer): BaseGCTransformer.__init__(self, parenttransformer.translator) diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -109,7 +109,7 @@ cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, gcpolicy=cls.gcpolicy) - db = cbuild.generate_graphs_for_llinterp() + db = cbuild.build_database() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph if option.view: @@ -1079,7 +1079,7 @@ def test_adr_of_nursery(self): run = self.runner("adr_of_nursery") res = run([]) - + class TestGenerationalNoFullCollectGC(GCTest): # test that nursery is doing its job and that no full collection @@ -1139,7 +1139,7 @@ 'large_object': 8*WORD, 'translated_to_c': False} root_stack_depth = 200 - + def define_ref_from_rawmalloced_to_regular(cls): import gc S = lltype.GcStruct('S', ('x', lltype.Signed)) @@ -1190,7 +1190,7 @@ run = self.runner("write_barrier_direct") res = run([]) assert res == 42 - + class TestMiniMarkGC(TestHybridGC): gcname = "minimark" GC_CAN_TEST_ID = True @@ -1207,7 +1207,7 @@ 'translated_to_c': False, } root_stack_depth = 200 - + def define_no_clean_setarrayitems(cls): # The optimization find_clean_setarrayitems() in # gctransformer/framework.py does not work with card marking. @@ -1232,7 +1232,7 @@ run = self.runner("no_clean_setarrayitems") res = run([]) assert res == 123 - + def define_nursery_hash_base(cls): class A: pass @@ -1291,19 +1291,19 @@ 'translated_to_c': False, } root_stack_depth = 200 - + def define_malloc_array_of_gcptr(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) A = lltype.GcArray(lltype.Ptr(S)) def f(): lst = lltype.malloc(A, 5) - return (lst[0] == lltype.nullptr(S) + return (lst[0] == lltype.nullptr(S) and lst[1] == lltype.nullptr(S) and lst[2] == lltype.nullptr(S) and lst[3] == lltype.nullptr(S) and lst[4] == lltype.nullptr(S)) return f - + def test_malloc_array_of_gcptr(self): run = self.runner('malloc_array_of_gcptr') res = run([]) @@ -1384,7 +1384,7 @@ def define_gettypeid(cls): class A(object): pass - + def fn(): a = A() return rgc.get_typeid(a) diff --git a/rpython/rlib/test/test_rpath.py b/rpython/rlib/test/test_rpath.py --- a/rpython/rlib/test/test_rpath.py +++ b/rpython/rlib/test/test_rpath.py @@ -68,8 +68,8 @@ assert rpath._nt_rabspath('d:\\foo\\bar\\..') == 'd:\\foo' assert rpath._nt_rabspath('d:\\foo\\bar\\..\\x') == 'd:\\foo\\x' curdrive = _ = rpath._nt_rsplitdrive(os.getcwd()) - assert len(curdrive) == 2 and curdrive[1] == ':' - assert rpath.rabspath('\\foo') == '%s\\foo' % curdrive + assert len(curdrive) == 2 and curdrive[0][1] == ':' + assert rpath.rabspath('\\foo') == '%s\\foo' % curdrive[0] def test_risabs_posix(): assert rpath._posix_risabs('/foo/bar') diff --git a/rpython/rtyper/extfunc.py b/rpython/rtyper/extfunc.py --- a/rpython/rtyper/extfunc.py +++ b/rpython/rtyper/extfunc.py @@ -47,6 +47,12 @@ impl = getattr(self, 'lltypeimpl', None) fakeimpl = getattr(self, 'lltypefakeimpl', self.instance) if impl: + if (rtyper.annotator.translator.config.translation.sandbox + and not self.safe_not_sandboxed): + from rpython.translator.sandbox.rsandbox import ( + make_sandbox_trampoline) + impl = make_sandbox_trampoline( + self.name, signature_args, s_result) if hasattr(self, 'lltypefakeimpl'): # If we have both an llimpl and an llfakeimpl, # we need a wrapper that selects the proper one and calls it @@ -75,14 +81,11 @@ return original_impl(%s) """ % (args, args, args)) in d impl = func_with_new_name(d['ll_wrapper'], name + '_wrapper') - if rtyper.annotator.translator.config.translation.sandbox: - impl._dont_inline_ = True # store some attributes to the 'impl' function, where # the eventual call to rtyper.getcallable() will find them # and transfer them to the final lltype.functionptr(). impl._llfnobjattrs_ = { '_name': self.name, - '_safe_not_sandboxed': self.safe_not_sandboxed, 'transactionsafe': self.transactionsafe } obj = rtyper.getannmixlevel().delayedfunction( diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -870,7 +870,7 @@ pinned = False if rgc.stm_is_enabled() or rgc.can_move(data): - if rgc.pin(data): + if not rgc.stm_is_enabled() and rgc.pin(data): pinned = True else: buf = lltype.malloc(TYPEP.TO, count, flavor='raw') diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -13,8 +13,9 @@ from rpython.rtyper.lltypesystem.lltype import ( Ptr, Struct, GcStruct, malloc, cast_pointer, castable, nullptr, RuntimeTypeInfo, getRuntimeTypeInfo, typeOf, Void, FuncType, Bool, Signed, - functionptr) + functionptr, attachRuntimeTypeInfo) from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.llannotation import SomePtr from rpython.rtyper.lltypesystem import rstr from rpython.rtyper.rmodel import ( Repr, getgcflavor, inputconst, warning, mangle) @@ -615,10 +616,17 @@ _callable=graph.func) else: destrptr = None - OBJECT = OBJECT_BY_FLAVOR[LLFLAVOR[self.gcflavor]] - self.rtyper.attachRuntimeTypeInfoFunc(self.object_type, - ll_runtime_type_info, - OBJECT, destrptr) + self.rtyper.call_all_setups() # compute ForwardReferences now + args_s = [SomePtr(Ptr(OBJECT))] + graph = self.rtyper.annotate_helper(ll_runtime_type_info, args_s) + s = self.rtyper.annotation(graph.getreturnvar()) + if (not isinstance(s, SomePtr) or + s.ll_ptrtype != Ptr(RuntimeTypeInfo)): + raise TyperError("runtime type info function returns %r, " + "expected Ptr(RuntimeTypeInfo)" % (s)) + funcptr = self.rtyper.getcallable(graph) + attachRuntimeTypeInfo(self.object_type, funcptr, destrptr) + vtable = self.rclass.getvtable() self.rtyper.set_type_for_typeptr(vtable, self.lowleveltype.TO) diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -1,14 +1,14 @@ """ RTyper: converts high-level operations into low-level operations in flow graphs. -The main class, with code to walk blocks and dispatch individual operations -to the care of the rtype_*() methods implemented in the other r* modules. -For each high-level operation 'hop', the rtype_*() methods produce low-level -operations that are collected in the 'llops' list defined here. When necessary, -conversions are inserted. +The main class, with code to walk blocks and dispatch individual operations to +the care of the rtype_*() methods implemented in the other r* modules. For +each high-level operation 'hop', the rtype_*() methods produce low-level +operations that are collected in the 'llops' list defined here. When +necessary, conversions are inserted. -This logic borrows a bit from rpython.annotator.annrpython, without the fixpoint -computation part. +This logic borrows a bit from rpython.annotator.annrpython, without the +fixpoint computation part. """ import os @@ -16,19 +16,20 @@ import py, math from rpython.annotator import model as annmodel, unaryop, binaryop -from rpython.rtyper.llannotation import SomePtr, lltype_to_annotation +from rpython.rtyper.llannotation import lltype_to_annotation from rpython.flowspace.model import Variable, Constant, SpaceOperation -from rpython.rtyper.annlowlevel import annotate_lowlevel_helper, LowLevelAnnotatorPolicy +from rpython.rtyper.annlowlevel import ( + annotate_lowlevel_helper, LowLevelAnnotatorPolicy) from rpython.rtyper.error import TyperError from rpython.rtyper.exceptiondata import ExceptionData from rpython.rtyper.lltypesystem.lltype import (Signed, Void, LowLevelType, - Ptr, ContainerType, FuncType, typeOf, RuntimeTypeInfo, - attachRuntimeTypeInfo, Primitive, getfunctionptr) -from rpython.rtyper.rmodel import Repr, inputconst, BrokenReprTyperError + ContainerType, typeOf, Primitive, getfunctionptr) +from rpython.rtyper.rmodel import Repr, inputconst from rpython.rtyper import rclass from rpython.rtyper.rclass import RootClassRepr from rpython.tool.pairtype import pair from rpython.translator.unsimplify import insert_empty_block +from rpython.translator.sandbox.rsandbox import make_sandbox_trampoline try: from pypystm import stmset, stmdict @@ -616,6 +617,17 @@ def getcallable(self, graph): def getconcretetype(v): return self.bindingrepr(v).lowleveltype + if self.annotator.translator.config.translation.sandbox: + try: + name = graph.func._sandbox_external_name + except AttributeError: + pass + else: + args_s = [v.annotation for v in graph.getargs()] + s_result = graph.getreturnvar().annotation + sandboxed = make_sandbox_trampoline(name, args_s, s_result) + return self.getannmixlevel().delayedfunction( + sandboxed, args_s, s_result) return getfunctionptr(graph, getconcretetype) @@ -645,21 +657,6 @@ graph = self.annotate_helper(ll_function, argtypes) return self.getcallable(graph) - def attachRuntimeTypeInfoFunc(self, GCSTRUCT, func, ARG_GCSTRUCT=None, - destrptr=None): - self.call_all_setups() # compute ForwardReferences now - if ARG_GCSTRUCT is None: - ARG_GCSTRUCT = GCSTRUCT - args_s = [SomePtr(Ptr(ARG_GCSTRUCT))] - graph = self.annotate_helper(func, args_s) - s = self.annotation(graph.getreturnvar()) - if (not isinstance(s, SomePtr) or - s.ll_ptrtype != Ptr(RuntimeTypeInfo)): - raise TyperError("runtime type info function %r returns %r, " - "excepted Ptr(RuntimeTypeInfo)" % (func, s)) - funcptr = self.getcallable(graph) - attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr) - # register operations from annotation model RPythonTyper._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from rpython.rtyper.lltypesystem.lltype import (Primitive, Ptr, typeOf, RuntimeTypeInfo, Struct, Array, FuncType, Void, ContainerType, OpaqueType, @@ -8,9 +9,9 @@ from rpython.rtyper.lltypesystem import llgroup from rpython.tool.sourcetools import valid_identifier from rpython.translator.c.primitive import PrimitiveName, PrimitiveType, name_gcref -from rpython.translator.c.node import StructDefNode, ArrayDefNode -from rpython.translator.c.node import FixedSizeArrayDefNode, BareBoneArrayDefNode -from rpython.translator.c.node import ContainerNodeFactory, ExtTypeOpaqueDefNode +from rpython.translator.c.node import ( + StructDefNode, ArrayDefNode, FixedSizeArrayDefNode, BareBoneArrayDefNode, + ContainerNodeFactory, ExtTypeOpaqueDefNode, FuncNode) from rpython.translator.c.support import cdecl, CNameManager from rpython.translator.c.support import log, barebonearray from rpython.translator.c.extfunc import do_the_getting @@ -28,6 +29,7 @@ def __init__(self, translator=None, standalone=False, gcpolicyclass=None, + exctransformer=None, thread_enabled=False, sandbox=False): self.translator = translator @@ -36,6 +38,7 @@ if gcpolicyclass is None: gcpolicyclass = gc.RefcountingGcPolicy self.gcpolicy = gcpolicyclass(self, thread_enabled) + self.exctransformer = exctransformer self.structdefnodes = {} self.pendingsetupnodes = [] @@ -45,7 +48,7 @@ self.delayedfunctionptrs = [] self.completedcontainers = 0 self.containerstats = {} - self.helper2ptr = {} + self.helpers = OrderedDict() # late_initializations is for when the value you want to # assign to a constant object is something C doesn't think is @@ -53,18 +56,14 @@ self.late_initializations = [] self.namespace = CNameManager() - if translator is None or translator.rtyper is None: - self.exctransformer = None - else: - self.exctransformer = translator.getexceptiontransformer() if translator is not None: - self.gctransformer = self.gcpolicy.gettransformer() + self.gctransformer = self.gcpolicy.gettransformer(translator) self.completed = False self.instrument_ncounter = 0 - self.with_stm = (self.translator is not None and - self.translator.config.translation.stm) + self.with_stm = (translator is not None and + translator.config.translation.stm) self.prebuilt_gc_counter = 0 def gettypedefnode(self, T, varlength=None): @@ -358,6 +357,8 @@ assert not self.delayedfunctionptrs self.completed = True + if self.gctransformer is not None and self.gctransformer.inline: + self.gctransformer.inline_helpers(self.all_graphs()) if show_progress: dump() log.database("Completed") @@ -389,30 +390,10 @@ produce(node) return result - def need_sandboxing(self, fnobj): - if not self.sandbox: - return False - if hasattr(fnobj, '_safe_not_sandboxed'): - return not fnobj._safe_not_sandboxed - elif getattr(getattr(fnobj, '_callable', None), - '_sandbox_external_name', None): - return True - else: - return "if_external" - - def prepare_inline_helpers(self): - all_nodes = self.globalcontainers() - funcnodes = [node for node in all_nodes if node.nodekind == 'func'] - graphs = [] - for node in funcnodes: - for graph in node.graphs_to_patch(): - graphs.append(graph) - self.gctransformer.prepare_inline_helpers(graphs) - def all_graphs(self): graphs = [] for node in self.containerlist: - if node.nodekind == 'func': + if isinstance(node, FuncNode): for graph in node.graphs_to_patch(): graphs.append(graph) return graphs diff --git a/rpython/translator/c/external.py b/rpython/translator/c/external.py deleted file mode 100644 --- a/rpython/translator/c/external.py +++ /dev/null @@ -1,54 +0,0 @@ -from rpython.rtyper.lltypesystem.lltype import typeOf, Void -from rpython.translator.c.support import USESLOTS # set to False if necessary while refactoring -from rpython.translator.c.support import cdecl, somelettersfrom - -class CExternalFunctionCodeGenerator(object): - if USESLOTS: - __slots__ = """db fnptr FUNCTYPE argtypenames resulttypename""".split() - - def __init__(self, fnptr, db): - self.fnptr = fnptr - self.db = db - self.FUNCTYPE = typeOf(fnptr) - assert Void not in self.FUNCTYPE.ARGS - self.argtypenames = [db.gettype(T) for T in self.FUNCTYPE.ARGS] - self.resulttypename = db.gettype(self.FUNCTYPE.RESULT) - - def graphs_to_patch(self): - return [] - - def name(self, cname): #virtual - return cname - - def argnames(self): - return ['%s%d' % (somelettersfrom(self.argtypenames[i]), i) - for i in range(len(self.argtypenames))] - - def allconstantvalues(self): - return [] - - def implementation_begin(self): - pass - - def cfunction_declarations(self): - if self.FUNCTYPE.RESULT is not Void: - yield '%s;' % cdecl(self.resulttypename, 'result') - - def cfunction_body(self): - try: - convert_params = self.fnptr.convert_params - except AttributeError: - convert_params = lambda backend, args: [arg for _,arg in args] - call = '%s(%s)' % (self.fnptr._name, ', '.join(convert_params("c", zip(self.FUNCTYPE.ARGS, self.argnames())))) - if self.FUNCTYPE.RESULT is not Void: - yield 'result = %s;' % call - yield 'if (PyErr_Occurred()) RPyConvertExceptionFromCPython();' - yield 'return result;' - else: - yield '%s;' % call - yield 'if (PyErr_Occurred()) RPyConvertExceptionFromCPython();' - - def implementation_end(self): - pass - -assert not USESLOTS or '__dict__' not in dir(CExternalFunctionCodeGenerator) diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py --- a/rpython/translator/c/extfunc.py +++ b/rpython/translator/c/extfunc.py @@ -1,23 +1,23 @@ import types from rpython.flowspace.model import FunctionGraph -from rpython.rtyper.lltypesystem import lltype, rstr, rlist +from rpython.annotator.listdef import s_list_of_strings +from rpython.rtyper.lltypesystem import lltype, rlist from rpython.rtyper.lltypesystem.rstr import STR, mallocstr from rpython.translator.c.support import cdecl def find_list_of_str(rtyper): - for r in rtyper.reprs.itervalues(): - if isinstance(r, rlist.ListRepr) and r.item_repr is rstr.string_repr: - return r.lowleveltype.TO - return None + r_strlist = rtyper.getrepr(s_list_of_strings) + rtyper.call_all_setups() + return r_strlist.lowleveltype.TO + def predeclare_common_types(db, rtyper): # Common types yield ('RPyString', STR) LIST_OF_STR = find_list_of_str(rtyper) - if LIST_OF_STR is not None: - yield ('RPyListOfString', LIST_OF_STR) + yield ('RPyListOfString', LIST_OF_STR) def predeclare_utility_functions(db, rtyper): # Common utility functions @@ -28,19 +28,18 @@ for fname, f in locals().items(): if isinstance(f, types.FunctionType): # XXX this is painful :( - if ("utility", fname) in db.helper2ptr: - yield (fname, db.helper2ptr["utility", fname]) + if ("utility", fname) in db.helpers: + yield (fname, db.helpers["utility", fname]) else: # hack: the defaults give the type of the arguments graph = rtyper.annotate_helper(f, f.func_defaults) - db.helper2ptr["utility", fname] = graph + db.helpers["utility", fname] = graph yield (fname, graph) -def predeclare_exception_data(db, rtyper): +def predeclare_exception_data(exctransformer, rtyper): # Exception-related types and constants exceptiondata = rtyper.exceptiondata - exctransformer = db.exctransformer yield ('RPYTHON_EXCEPTION_VTABLE', exceptiondata.lltype_of_exception_type) yield ('RPYTHON_EXCEPTION', exceptiondata.lltype_of_exception_value) @@ -68,19 +67,19 @@ def predeclare_all(db, rtyper): for fn in [predeclare_common_types, predeclare_utility_functions, - predeclare_exception_data, ]: for t in fn(db, rtyper): yield t + exctransformer = db.exctransformer + for t in predeclare_exception_data(exctransformer, rtyper): + yield t + def get_all(db, rtyper): - for fn in [predeclare_common_types, - predeclare_utility_functions, - predeclare_exception_data, - ]: - for t in fn(db, rtyper): - yield t[1] + for name, fnptr in predeclare_all(db, rtyper): + yield fnptr + # ____________________________________________________________ diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -2,7 +2,7 @@ from rpython.translator.c.support import cdecl from rpython.translator.c.support import llvalue_from_constant, gen_assignments from rpython.translator.c.support import c_string_constant, barebonearray -from rpython.flowspace.model import Variable, Constant, copygraph +from rpython.flowspace.model import Variable, Constant from rpython.rtyper.lltypesystem.lltype import (Ptr, Void, Bool, Signed, Unsigned, SignedLongLong, Float, UnsignedLongLong, Char, UniChar, ContainerType, Array, FixedSizeArray, ForwardReference, FuncType) @@ -18,6 +18,16 @@ KEEP_INLINED_GRAPHS = False +def make_funcgen(graph, db, exception_policy, functionname): + graph._seen_by_the_backend = True + # apply the exception transformation + if db.exctransformer: + db.exctransformer.create_exception_handling(graph) + # apply the gc transformation + if db.gctransformer: + db.gctransformer.transform_graph(graph) + return FunctionCodeGenerator(graph, db, exception_policy, functionname) + class FunctionCodeGenerator(object): """ Collects information about a function which we have to generate @@ -25,22 +35,14 @@ """ use_stm_rewind_jmp_frame = False - def __init__(self, graph, db, exception_policy=None, functionname=None): - graph._seen_by_the_backend = True + def __init__(self, graph, db, exception_policy, functionname): self.graph = graph self.db = db self.gcpolicy = db.gcpolicy self.exception_policy = exception_policy self.functionname = functionname - # apply the exception transformation - if self.db.exctransformer: - self.db.exctransformer.create_exception_handling(self.graph) - # apply the gc transformation - if self.db.gctransformer: - self.db.gctransformer.transform_graph(self.graph) - #self.graph.show() + self.collect_var_and_types() - for v in self.vars: T = v.concretetype # obscure: skip forward references and hope for the best @@ -78,12 +80,6 @@ self.more_ll_values.append(link.llexitcase) elif link.exitcase is not None: mix.append(Constant(link.exitcase)) - if self.exception_policy == "CPython": - v, exc_cleanup_ops = self.graph.exc_cleanup - mix.append(v) - for cleanupop in exc_cleanup_ops: - mix.extend(cleanupop.args) - mix.append(cleanupop.result) uniquemix = [] seen = identity_dict() @@ -93,20 +89,7 @@ seen[v] = True self.vars = uniquemix - def name(self, cname): #virtual - return cname - - def patch_graph(self, copy_graph): - graph = self.graph - if self.db.gctransformer and self.db.gctransformer.inline: - if copy_graph: - graph = copygraph(graph, shallow=True) - self.db.gctransformer.inline_helpers(graph) - return graph - def implementation_begin(self): - self.oldgraph = self.graph - self.graph = self.patch_graph(copy_graph=True) SSI_to_SSA(self.graph) self.collect_var_and_types() self.blocknum = {} @@ -132,8 +115,6 @@ self.vars = None self.blocknum = None self.innerloops = None - self.graph = self.oldgraph - del self.oldgraph def argnames(self): return [LOCALVAR % v.name for v in self.graph.getargs()] @@ -251,8 +232,6 @@ yield '}' link = block.exits[0] assert link.exitcase in (False, True) - #yield 'assert(%s == %s);' % (self.expr(block.exitswitch), - # self.genc.nameofvalue(link.exitcase, ct)) for op in self.gen_link(link): yield op elif TYPE in (Signed, Unsigned, SignedLongLong, @@ -941,14 +920,11 @@ def getdebugfunctionname(self): name = self.functionname - if not name: - return "?" if name.startswith('pypy_g_'): name = name[7:] return name def OP_DEBUG_RECORD_TRACEBACK(self, op): - #if self.functionname is None, we print "?" as the argument */ return 'PYPY_DEBUG_RECORD_TRACEBACK("%s");' % ( self.getdebugfunctionname(),) diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -1,8 +1,7 @@ import sys from rpython.flowspace.model import Constant -from rpython.rtyper.lltypesystem import lltype -from rpython.rtyper.lltypesystem.lltype import (typeOf, RttiStruct, - RuntimeTypeInfo, top_container) +from rpython.rtyper.lltypesystem.lltype import (RttiStruct, + RuntimeTypeInfo) from rpython.translator.c.node import ContainerNode from rpython.translator.c.support import cdecl from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -18,23 +17,12 @@ return defnode.db.gctransformer.HDR return None - def common_gcheader_initdata(self, defnode): - if defnode.db.gctransformer is not None: - raise NotImplementedError - return None - def struct_gcheader_definition(self, defnode): return self.common_gcheader_definition(defnode) - def struct_gcheader_initdata(self, defnode): - return self.common_gcheader_initdata(defnode) - def array_gcheader_definition(self, defnode): return self.common_gcheader_definition(defnode) - def array_gcheader_initdata(self, defnode): - return self.common_gcheader_initdata(defnode) - def compilation_info(self): if not self.db: return ExternalCompilationInfo() @@ -46,9 +34,6 @@ ] ) - def get_prebuilt_hash(self, obj): - return None - def need_no_typeptr(self): return False @@ -109,16 +94,9 @@ class RefcountingGcPolicy(BasicGcPolicy): - def gettransformer(self): + def gettransformer(self, translator): from rpython.memory.gctransform import refcounting - return refcounting.RefcountingGCTransformer(self.db.translator) - - def common_gcheader_initdata(self, defnode): - if defnode.db.gctransformer is not None: - gct = defnode.db.gctransformer - top = top_container(defnode.obj) - return gct.gcheaderbuilder.header_of_object(top)._obj - return None + return refcounting.RefcountingGCTransformer(translator) # for structs @@ -197,18 +175,9 @@ class BoehmGcPolicy(BasicGcPolicy): - def gettransformer(self): + def gettransformer(self, translator): from rpython.memory.gctransform import boehm - return boehm.BoehmGCTransformer(self.db.translator) - - def common_gcheader_initdata(self, defnode): - if defnode.db.gctransformer is not None: - ptr = defnode.obj._as_ptr() - hdr = lltype.malloc(defnode.db.gctransformer.HDR, immortal=True) - hdr.hash = lltype.identityhash_nocache(ptr) - defnode.db.gctransformer.gcheader_initdata(hdr, ptr) - return hdr._obj - return None + return boehm.BoehmGCTransformer(translator) def array_setup(self, arraydefnode): pass @@ -321,9 +290,9 @@ class BasicFrameworkGcPolicy(BasicGcPolicy): - def gettransformer(self): + def gettransformer(self, translator): if hasattr(self, 'transformerclass'): # for rpython/memory tests - return self.transformerclass(self.db.translator) + return self.transformerclass(translator) raise NotImplementedError def struct_setup(self, structdefnode, rtti): @@ -372,24 +341,6 @@ args = [funcgen.expr(v) for v in op.args] return '%s = %s; /* for moving GCs */' % (args[1], args[0]) - def common_gcheader_initdata(self, defnode): - o = top_container(defnode.obj) - needs_hash = self.get_prebuilt_hash(o) is not None - hdr = defnode.db.gctransformer.gc_header_for(o, needs_hash) - return hdr._obj - - def get_prebuilt_hash(self, obj): - # for prebuilt objects that need to have their hash stored and - # restored. Note that only structures that are StructNodes all - # the way have their hash stored (and not e.g. structs with var- - # sized arrays at the end). 'obj' must be the top_container. - TYPE = typeOf(obj) - if not isinstance(TYPE, lltype.GcStruct): - return None - if TYPE._is_varsize(): - return None - return getattr(obj, '_hash_cache_', None) - def need_no_typeptr(self): config = self.db.translator.config return config.translation.gcremovetypeptr @@ -450,15 +401,15 @@ class ShadowStackFrameworkGcPolicy(BasicFrameworkGcPolicy): - def gettransformer(self): + def gettransformer(self, translator): from rpython.memory.gctransform import shadowstack - return shadowstack.ShadowStackFrameworkGCTransformer(self.db.translator) + return shadowstack.ShadowStackFrameworkGCTransformer(translator) class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy): - def gettransformer(self): + def gettransformer(self, translator): from rpython.memory.gctransform import asmgcroot - return asmgcroot.AsmGcRootFrameworkGCTransformer(self.db.translator) + return asmgcroot.AsmGcRootFrameworkGCTransformer(translator) def GC_KEEPALIVE(self, funcgen, v): return 'pypy_asm_keepalive(%s);' % funcgen.expr(v) @@ -468,18 +419,9 @@ class StmFrameworkGcPolicy(BasicFrameworkGcPolicy): - def gettransformer(self): + def gettransformer(self, translator): from rpython.memory.gctransform import stmframework - return stmframework.StmFrameworkGCTransformer(self.db.translator) - - def get_prebuilt_hash(self, obj): - return None # done differently with the stmgc - - def get_stm_prebuilt_hash(self, obj): - h = BasicFrameworkGcPolicy.get_prebuilt_hash(self, obj) - if h is None: - h = lltype.identityhash(obj._as_ptr()) - return h + return stmframework.StmFrameworkGCTransformer(translator) name_to_gcpolicy = { @@ -490,3 +432,5 @@ 'framework+asmgcc': AsmGcRootFrameworkGcPolicy, 'framework+stm': StmFrameworkGcPolicy, } + + diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -134,8 +134,10 @@ if not self.standalone: raise NotImplementedError("--gcrootfinder=asmgcc requires standalone") + exctransformer = translator.getexceptiontransformer() db = LowLevelDatabase(translator, standalone=self.standalone, gcpolicyclass=gcpolicyclass, + exctransformer=exctransformer, thread_enabled=self.config.translation.thread, sandbox=self.config.translation.sandbox) self.db = db @@ -212,22 +214,8 @@ 'RPY_LL_ASSERT': 1, 'RPY_STM_ASSERT': 1} - def generate_graphs_for_llinterp(self, db=None): - # prepare the graphs as when the source is generated, but without - # actually generating the source. - if db is None: - db = self.build_database() - graphs = db.all_graphs() - db.gctransformer.prepare_inline_helpers(graphs) - for node in db.containerlist: - if hasattr(node, 'funcgens'): - for funcgen in node.funcgens: - funcgen.patch_graph(copy_graph=False) - return db - def generate_source(self, db=None, defines={}, exe_name=None): assert self.c_source_filename is None - if db is None: db = self.build_database() pf = self.getentrypointptr() @@ -886,7 +874,7 @@ print >> f, '};' print >> f, 'static long rpy_prebuilt_hashes[] = {' for _, node in gclist: - h = database.gcpolicy.get_stm_prebuilt_hash(node.obj) + h = database.gctransformer.get_stm_prebuilt_hash(node.obj) print >> f, '\t%s,' % (name_signed(h, database),) print >> f, '};' print >> f @@ -967,7 +955,6 @@ # sg = SourceGenerator(database) sg.set_strategy(targetdir, split) - database.prepare_inline_helpers() sg.gen_readable_parts_of_source(f) headers_to_precompile = sg.headers_to_precompile[:] headers_to_precompile.insert(0, incfilename) diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py --- a/rpython/translator/c/node.py +++ b/rpython/translator/c/node.py @@ -3,8 +3,7 @@ Void, OpaqueType, Float, RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray) from rpython.rtyper.lltypesystem import llmemory, llgroup -from rpython.translator.c.funcgen import FunctionCodeGenerator -from rpython.translator.c.external import CExternalFunctionCodeGenerator +from rpython.translator.c.funcgen import make_funcgen from rpython.translator.c.support import USESLOTS # set to False if necessary while refactoring from rpython.translator.c.support import cdecl, forward_cdecl, somelettersfrom from rpython.translator.c.support import c_char_array_constant, barebonearray @@ -594,7 +593,17 @@ class StructNode(ContainerNode): nodekind = 'struct' if USESLOTS: - __slots__ = ('is_weakref',) + __slots__ = ('gc_init', 'is_weakref') + + def __init__(self, db, T, obj): + ContainerNode.__init__(self, db, T, obj) + if needs_gcheader(T): + gct = self.db.gctransformer + if gct is not None: + self.gc_init = gct.gcheader_initdata(self) + db.getcontainernode(self.gc_init) + else: + self.gc_init = None def basename(self): T = self.getTYPE() @@ -621,8 +630,7 @@ data = [] if needs_gcheader(T): - gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) - data.append(('gcheader', gc_init)) + data.append(('gcheader', self.gc_init)) for name in defnode.fieldnames: data.append((name, getattr(self.obj, name))) @@ -695,7 +703,7 @@ def implementation(self): hash_typename = self.get_hash_typename() - hash = self.db.gcpolicy.get_prebuilt_hash(self.obj) + hash = self.db.gctransformer.get_prebuilt_hash(self.obj) assert hash is not None lines = list(self.initializationexpr()) lines.insert(0, '%s = { {' % ( @@ -705,7 +713,8 @@ return lines def gcstructnode_factory(db, T, obj): - if db.gcpolicy.get_prebuilt_hash(obj) is not None: + if (db.gctransformer and + db.gctransformer.get_prebuilt_hash(obj) is not None): cls = GcStructNodeWithHash else: cls = StructNode @@ -715,7 +724,17 @@ class ArrayNode(ContainerNode): nodekind = 'array' if USESLOTS: - __slots__ = () + __slots__ = ('gc_init',) + + def __init__(self, db, T, obj): + ContainerNode.__init__(self, db, T, obj) + if needs_gcheader(T): + gct = self.db.gctransformer + if gct is not None: + self.gc_init = gct.gcheader_initdata(self) + db.getcontainernode(self.gc_init) + else: + self.gc_init = None def getptrname(self, static=False): if barebonearray(self.getTYPE()): @@ -735,8 +754,7 @@ T = self.getTYPE() yield '{' if needs_gcheader(T): - gc_init = self.db.gcpolicy.array_gcheader_initdata(self) - lines = generic_initializationexpr(self.db, gc_init, 'gcheader', + lines = generic_initializationexpr(self.db, self.gc_init, 'gcheader', '%sgcheader' % (decoration,)) for line in lines: yield line @@ -836,81 +854,64 @@ comma = '' expr += comma i = expr.find('\n') - if i<0: i = len(expr) + if i < 0: + i = len(expr) expr = '%s\t/* %s */%s' % (expr[:i], decoration, expr[i:]) return expr.split('\n') # ____________________________________________________________ -class FuncNode(ContainerNode): +class FuncNodeBase(ContainerNode): nodekind = 'func' eci_name = 'compilation_info' # there not so many node of this kind, slots should not # be necessary - - def __init__(self, db, T, obj, forcename=None): + def __init__(self, db, T, obj, ptrname): Node.__init__(self, db) self.globalcontainer = True self.T = T self.obj = obj - callable = getattr(obj, '_callable', None) - if (callable is not None and - getattr(callable, 'c_name', None) is not None): - self.name = forcename or obj._callable.c_name - elif getattr(obj, 'external', None) == 'C' and not db.need_sandboxing(obj): - self.name = forcename or self.basename() - else: - self.name = (forcename or - db.namespace.uniquename('g_' + self.basename())) - self.make_funcgens() + self.name = ptrname self.typename = db.gettype(T) #, who_asks=self) def getptrname(self, static=False): return self.name - def make_funcgens(self): - self.funcgens = select_function_code_generators(self.obj, self.db, self.name) - if self.funcgens: - argnames = self.funcgens[0].argnames() #Assume identical for all funcgens - self.implementationtypename = self.db.gettype(self.T, argnames=argnames) - self._funccodegen_owner = self.funcgens[0] - else: - self._funccodegen_owner = None - def basename(self): return self.obj._name + +class FuncNode(FuncNodeBase): + def __init__(self, db, T, obj, ptrname): + FuncNodeBase.__init__(self, db, T, obj, ptrname) + exception_policy = getattr(obj, 'exception_policy', None) + self.funcgen = make_funcgen(obj.graph, db, exception_policy, ptrname) + argnames = self.funcgen.argnames() + self.implementationtypename = db.gettype(T, argnames=argnames) + self._funccodegen_owner = self.funcgen + def enum_dependencies(self): - if not self.funcgens: - return [] - return self.funcgens[0].allconstantvalues() #Assume identical for all funcgens + return self.funcgen.allconstantvalues() def forward_declaration(self): callable = getattr(self.obj, '_callable', None) is_exported = getattr(callable, 'exported_symbol', False) - for funcgen in self.funcgens: - yield '%s;' % ( - forward_cdecl(self.implementationtypename, - funcgen.name(self.name), self.db.standalone, - is_exported=is_exported)) + yield '%s;' % ( + forward_cdecl(self.implementationtypename, + self.name, self.db.standalone, is_exported=is_exported)) + + def graphs_to_patch(self): + for i in self.funcgen.graphs_to_patch(): + yield i def implementation(self): - for funcgen in self.funcgens: - for s in self.funcgen_implementation(funcgen): - yield s - - def graphs_to_patch(self): - for funcgen in self.funcgens: - for i in funcgen.graphs_to_patch(): - yield i - - def funcgen_implementation(self, funcgen): + funcgen = self.funcgen funcgen.implementation_begin() # recompute implementationtypename as the argnames may have changed argnames = funcgen.argnames() implementationtypename = self.db.gettype(self.T, argnames=argnames) - yield '%s {' % cdecl(implementationtypename, funcgen.name(self.name)) + yield '%s {' % cdecl(implementationtypename, self.name) # # declare the local variables # @@ -921,7 +922,7 @@ while start < len(localnames): # pack the local declarations over as few lines as possible total = lengths[start] + 8 - end = start+1 + end = start + 1 while total + lengths[end] < 77: total += lengths[end] + 1 end += 1 @@ -952,44 +953,55 @@ del bodyiter funcgen.implementation_end() -def sandbox_stub(fnobj, db): - # unexpected external function for --sandbox translation: replace it - # with a "Not Implemented" stub. To support these functions, port them - # to the new style registry (e.g. rpython.module.ll_os.RegisterOs). - from rpython.translator.sandbox import rsandbox - graph = rsandbox.get_external_function_sandbox_graph(fnobj, db, - force_stub=True) - return [FunctionCodeGenerator(graph, db)] +class ExternalFuncNode(FuncNodeBase): + def __init__(self, db, T, obj, ptrname): + FuncNodeBase.__init__(self, db, T, obj, ptrname) + self._funccodegen_owner = None -def sandbox_transform(fnobj, db): - # for --sandbox: replace a function like os_open_llimpl() with - # code that communicates with the external process to ask it to - # perform the operation. - from rpython.translator.sandbox import rsandbox - graph = rsandbox.get_external_function_sandbox_graph(fnobj, db) - return [FunctionCodeGenerator(graph, db)] + def enum_dependencies(self): + return [] -def select_function_code_generators(fnobj, db, functionname): - sandbox = db.need_sandboxing(fnobj) - if hasattr(fnobj, 'graph'): - if sandbox and sandbox != "if_external": - # apply the sandbox transformation - return sandbox_transform(fnobj, db) - exception_policy = getattr(fnobj, 'exception_policy', None) - return [FunctionCodeGenerator(fnobj.graph, db, exception_policy, - functionname)] - elif getattr(fnobj, 'external', None) is not None: - if sandbox: - return sandbox_stub(fnobj, db) - elif fnobj.external == 'C': - return [] - else: - assert fnobj.external == 'CPython' - return [CExternalFunctionCodeGenerator(fnobj, db)] - elif hasattr(fnobj._callable, "c_name"): - return [] # this case should only be used for entrypoints + def forward_declaration(self): + return [] + + def implementation(self): + return [] + +def new_funcnode(db, T, obj, forcename=None): + if db.sandbox: + if (getattr(obj, 'external', None) is not None and + not obj._safe_not_sandboxed): + from rpython.translator.sandbox import rsandbox + obj.__dict__['graph'] = rsandbox.get_sandbox_stub( + obj, db.translator.rtyper) + obj.__dict__.pop('_safe_not_sandboxed', None) + obj.__dict__.pop('external', None) + if forcename: + name = forcename else: - raise ValueError("don't know how to generate code for %r" % (fnobj,)) + name = _select_name(db, obj) + if hasattr(obj, 'graph'): + return FuncNode(db, T, obj, name) + elif getattr(obj, 'external', None) is not None: + assert obj.external == 'C' + if db.sandbox: + assert obj._safe_not_sandboxed + return ExternalFuncNode(db, T, obj, name) + elif hasattr(obj._callable, "c_name"): + return ExternalFuncNode(db, T, obj, name) # this case should only be used for entrypoints + else: + raise ValueError("don't know how to generate code for %r" % (obj,)) + + +def _select_name(db, obj): + try: + return obj._callable.c_name + except AttributeError: + pass + if getattr(obj, 'external', None) == 'C': + return obj._name + return db.namespace.uniquename('g_' + obj._name) + class ExtType_OpaqueNode(ContainerNode): nodekind = 'rpyopaque' @@ -1104,7 +1116,7 @@ Array: ArrayNode, GcArray: ArrayNode, FixedSizeArray: FixedSizeArrayNode, - FuncType: FuncNode, + FuncType: new_funcnode, OpaqueType: opaquenode_factory, llmemory._WeakRefType: weakrefnode_factory, llgroup.GroupType: GroupNode, diff --git a/rpython/translator/c/test/test_database.py b/rpython/translator/c/test/test_database.py --- a/rpython/translator/c/test/test_database.py +++ b/rpython/translator/c/test/test_database.py @@ -9,8 +9,6 @@ def dump_on_stdout(database): - if database.gctransformer: - database.prepare_inline_helpers() print '/*********************************/' structdeflist = database.getstructdeflist() for node in structdeflist: @@ -171,7 +169,7 @@ F = FuncType([Signed], Signed) f = functionptr(F, "f", graph=graph) - db = LowLevelDatabase(t) + db = LowLevelDatabase(t, exctransformer=t.getexceptiontransformer()) db.get(f) db.complete() dump_on_stdout(db) @@ -186,7 +184,7 @@ return p.x * p.y t, graph = makegraph(ll_f, [int]) - db = LowLevelDatabase(t) + db = LowLevelDatabase(t, exctransformer=t.getexceptiontransformer()) db.get(getfunctionptr(graph)) db.complete() dump_on_stdout(db) @@ -207,7 +205,7 @@ return s.ptr1.x * s.ptr2.x t, graph = makegraph(ll_f, [int]) - db = LowLevelDatabase(t) + db = LowLevelDatabase(t, exctransformer=t.getexceptiontransformer()) db.get(getfunctionptr(graph)) db.complete() dump_on_stdout(db) diff --git a/rpython/translator/c/test/test_refcount.py b/rpython/translator/c/test/test_refcount.py --- a/rpython/translator/c/test/test_refcount.py +++ b/rpython/translator/c/test/test_refcount.py @@ -106,37 +106,6 @@ assert fn(1) == 4 assert fn(0) == 5 - def test_del_basic(self): - py.test.skip("xxx fix or kill") - S = lltype.GcStruct('S', ('x', lltype.Signed), rtti=True) - TRASH = lltype.GcStruct('TRASH', ('x', lltype.Signed)) - GLOBAL = lltype.Struct('GLOBAL', ('x', lltype.Signed)) - glob = lltype.malloc(GLOBAL, immortal=True) - def destructor(s): - glob.x = s.x + 1 - def type_info_S(s): - return lltype.getRuntimeTypeInfo(S) - - def g(n): - s = lltype.malloc(S) - s.x = n - # now 's' should go away - def entrypoint(n): - g(n) - # llop.gc__collect(lltype.Void) - return glob.x - - t = TranslationContext() - t.buildannotator().build_types(entrypoint, [int]) - rtyper = t.buildrtyper() - destrptr = rtyper.annotate_helper_fn(destructor, [lltype.Ptr(S)]) - rtyper.attachRuntimeTypeInfoFunc(S, type_info_S, destrptr=destrptr) - rtyper.specialize() - fn = self.compile_func(entrypoint, None, t) - - res = fn(123) - assert res == 124 - def test_del_catches(self): import os def g(): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -151,7 +151,7 @@ # Increase stack size, for the linker and the stack check code. stack_size = 8 << 20 # 8 Mb - self.link_flags.append('/STACK:%d' % stack_size) + self.link_flags = self.link_flags + ('/STACK:%d' % stack_size,) # The following symbol is used in c/src/stack.h self.cflags.append('/DMAX_STACK_SIZE=%d' % (stack_size - 1024)) diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py --- a/rpython/translator/sandbox/rsandbox.py +++ b/rpython/translator/sandbox/rsandbox.py @@ -16,7 +16,6 @@ from rpython.rlib import rposix from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.llannotation import lltype_to_annotation -from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.tool.ansi_print import ansi_log @@ -37,7 +36,8 @@ sandboxsafe=True) -@signature(types.int(), types.ptr(rffi.CCHARP.TO), types.int(), returns=types.none()) +@signature(types.int(), types.ptr(rffi.CCHARP.TO), types.int(), + returns=types.none()) def writeall_not_sandboxed(fd, buf, length): while length > 0: size = rffi.cast(rffi.SIZE_T, length) @@ -85,15 +85,24 @@ return loader def reraise_error(error, loader): - if error == 1: raise OSError(load_int(loader), "external error") - elif error == 2: raise IOError - elif error == 3: raise OverflowError - elif error == 4: raise ValueError - elif error == 5: raise ZeroDivisionError - elif error == 6: raise MemoryError - elif error == 7: raise KeyError - elif error == 8: raise IndexError - else: raise RuntimeError + if error == 1: + raise OSError(load_int(loader), "external error") + elif error == 2: + raise IOError + elif error == 3: + raise OverflowError + elif error == 4: + raise ValueError + elif error == 5: + raise ZeroDivisionError + elif error == 6: + raise MemoryError + elif error == 7: + raise KeyError + elif error == 8: + raise IndexError + else: + raise RuntimeError @signature(types.str(), returns=types.impossible()) @@ -101,51 +110,46 @@ STDERR = 2 with rffi.scoped_str2charp(msg + '\n') as buf: writeall_not_sandboxed(STDERR, buf, len(msg) + 1) - raise RuntimeError(msg) # XXX in RPython, the msg is ignored at the moment + raise RuntimeError(msg) # XXX in RPython, the msg is ignored + +def make_stub(fnname, msg): + """Build always-raising stub function to replace unsupported external.""" + log.WARNING(msg) + + def execute(*args): + not_implemented_stub(msg) + execute.__name__ = 'sandboxed_%s' % (fnname,) + return execute + +def sig_ll(fnobj): + FUNCTYPE = lltype.typeOf(fnobj) + args_s = [lltype_to_annotation(ARG) for ARG in FUNCTYPE.ARGS] + s_result = lltype_to_annotation(FUNCTYPE.RESULT) + return args_s, s_result dump_string = rmarshal.get_marshaller(str) -load_int = rmarshal.get_loader(int) +load_int = rmarshal.get_loader(int) -def get_external_function_sandbox_graph(fnobj, db, force_stub=False): - """Build the graph of a helper trampoline function to be used - in place of real calls to the external function 'fnobj'. The - trampoline marshals its input arguments, dumps them to STDOUT, - and waits for an answer on STDIN. +def get_sandbox_stub(fnobj, rtyper): + fnname = fnobj._name + args_s, s_result = sig_ll(fnobj) + msg = "Not implemented: sandboxing for external function '%s'" % (fnname,) + execute = make_stub(fnname, msg) + return _annotate(rtyper, execute, args_s, s_result) + +def make_sandbox_trampoline(fnname, args_s, s_result): + """Create a trampoline function with the specified signature. + + The trampoline is meant to be used in place of real calls to the external + function named 'fnname'. It marshals its input arguments, dumps them to + STDOUT, and waits for an answer on STDIN. """ - if getattr(getattr(fnobj, '_callable', None), - '_sandbox_external_name', None): - fnname = fnobj._callable._sandbox_external_name - else: - fnname = fnobj._name - if hasattr(fnobj, 'graph'): - # get the annotation of the input arguments and the result - graph = fnobj.graph - annotator = db.translator.annotator - args_s = [annotator.binding(v) for v in graph.getargs()] - s_result = annotator.binding(graph.getreturnvar()) - else: - # pure external function - fall back to the annotations - # corresponding to the ll types - FUNCTYPE = lltype.typeOf(fnobj) - args_s = [lltype_to_annotation(ARG) for ARG in FUNCTYPE.ARGS] - s_result = lltype_to_annotation(FUNCTYPE.RESULT) - try: - if force_stub: # old case - don't try to support suggested_primitive - raise NotImplementedError("sandboxing for external function '%s'" - % (fnname,)) - dump_arguments = rmarshal.get_marshaller(tuple(args_s)) load_result = rmarshal.get_loader(s_result) - - except (NotImplementedError, - rmarshal.CannotMarshal, - rmarshal.CannotUnmarshall), e: - msg = 'Not Implemented: %s' % (e,) - log.WARNING(msg) - def execute(*args): - not_implemented_stub(msg) - + except (rmarshal.CannotMarshal, rmarshal.CannotUnmarshall) as e: + msg = "Cannot sandbox function '%s': %s" % (fnname, e) + execute = make_stub(fnname, msg) else: def execute(*args): # marshal the function name and input arguments @@ -158,9 +162,12 @@ result = load_result(loader) loader.check_finished() return result - execute = func_with_new_name(execute, 'sandboxed_' + fnname) + execute.__name__ = 'sandboxed_%s' % (fnname,) + return execute - ann = MixLevelHelperAnnotator(db.translator.rtyper) - graph = ann.getgraph(execute, args_s, s_result) + +def _annotate(rtyper, f, args_s, s_result): + ann = MixLevelHelperAnnotator(rtyper) + graph = ann.getgraph(f, args_s, s_result) ann.finish() return graph diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -33,15 +33,19 @@ def __init__(self, db, T, obj): assert isinstance(obj._name, int) - self.db = db - self.T = T - self.obj = obj + ContainerNode.__init__(self, db, T, obj) def initializationexpr(self, decoration=''): yield '{ { }, %s }' % ( name_small_integer(self.obj.typeid16, self.db)) # self.obj.prebuilt_hash + def enum_dependencies(self): + return [] + + def basename(self): + return self.nodekind + def stm_hint_commit_soon(funcgen, op): return ('if (!stm_is_atomic(&stm_thread_local))\n' _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit