Author: Armin Rigo <ar...@tunes.org> Branch: py3.5 Changeset: r90367:4618cb6fc961 Date: 2017-02-26 18:10 +0100 http://bitbucket.org/pypy/pypy/changeset/4618cb6fc961/
Log: hg merge default diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -156,19 +156,30 @@ Does PyPy have a GIL? Why? ------------------------------------------------- -Yes, PyPy has a GIL. Removing the GIL is very hard. The problems are -essentially the same as with CPython (including the fact that our -garbage collectors are not thread-safe so far). Fixing it is possible, -as shown by Jython and IronPython, but difficult. It would require -adapting the whole source code of PyPy, including subtle decisions about -whether some effects are ok or not for the user (i.e. the Python -programmer). +Yes, PyPy has a GIL. Removing the GIL is very hard. On top of CPython, +you have two problems: (1) GC, in this case reference counting; (2) the +whole Python language. -Instead, since 2012, there is work going on on a still very experimental -:doc:`Software Transactional Memory <stm>` (STM) version of PyPy. This should give -an alternative PyPy which works without a GIL, while at the same time -continuing to give the Python programmer the complete illusion of having -one. +For PyPy, the hard issue is (2): by that I mean issues like what occurs +if a mutable object is changed from one thread and read from another +concurrently. This is a problem for *any* mutable type: it needs +careful review and fixes (fine-grained locks, mostly) through the +*whole* Python interpreter. It is a major effort, although not +completely impossible, as Jython/IronPython showed. This includes +subtle decisions about whether some effects are ok or not for the user +(i.e. the Python programmer). + +CPython has additionally the problem (1) of reference counting. With +PyPy, this sub-problem is simpler: we need to make our GC +multithread-aware. This is easier to do efficiently in PyPy than in +CPython. It doesn't solve the issue (2), though. + +Note that since 2012 there is work going on on a still very experimental +:doc:`Software Transactional Memory <stm>` (STM) version of PyPy. This +should give an alternative PyPy which works without a GIL, while at the +same time continuing to give the Python programmer the complete illusion +of having one. This work is currently a bit stalled because of its own +technical difficulties. Is PyPy more clever than CPython about Tail Calls? diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -152,6 +152,13 @@ behavior is also available on Python 2.x or for the ``dict`` type by calling ``__pypy__.move_to_end(dict, key, last=True)``. + +.. branch optinfo-into-bridges-3 + +Improve the optimization of branchy Python code by retaining more information +across failing guards. + + .. branch: space-newtext Internal refactoring of ``space.wrap()``, which is now replaced with diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -781,6 +781,19 @@ else: raise Exception("DID NOT RAISE") + def test_indent_error_filename(self): + source = py.code.Source(""" + def f(): + x + y + """) + try: + self.simple_test(source, None, None) + except IndentationError as e: + assert e.filename == '<test>' + else: + raise Exception("DID NOT RAISE") + def test_kwargs_last(self): py.test.raises(SyntaxError, self.simple_test, "int(base=10, '2')", None, None) diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -205,6 +205,9 @@ except error.TokenError as e: e.filename = compile_info.filename raise + except error.TokenIndentationError as e: + e.filename = compile_info.filename + raise except parser.ParseError as e: # Catch parse errors, pretty them up and reraise them as a # SyntaxError. diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -77,7 +77,7 @@ """) assert self.space.unwrap(w_args) == ( 'unindent does not match any outer indentation level', - (None, 3, 0, ' y\n')) + ('<string>', 3, 0, ' y\n')) def test_getcodeflags(self): code = self.compiler.compile('from __future__ import division\n', @@ -1334,6 +1334,18 @@ else: raise Exception("DID NOT RAISE") + def test_outdentation_error_filename(self): + source = """if 1: + x + y + """ + try: + exec(source) + except IndentationError as e: + assert e.filename == '<string>' + else: + raise Exception("DID NOT RAISE") + def test_taberror(self): source = """if 1: x diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -289,8 +289,9 @@ # If L, V -> LV current = SBase + ((current - LBase)*VCount + (next - VBase)) * TCount continue + # Note: if next == TBase, leave LV unchanged if (SBase <= current < SBase + SCount and - TBase <= next < TBase + TCount and + TBase < next < TBase + TCount and (current - SBase) % TCount == 0): # If LV, T -> LVT current = current + (next - TBase) diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py new file mode 100644 --- /dev/null +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -0,0 +1,47 @@ +import pytest +try: + from hypothesis import given, strategies as st, example, settings +except ImportError: + pytest.skip("hypothesis required") + +from pypy.module.unicodedata.interp_ucd import ucd + +def make_normalization(space, NF_code): + def normalize(s): + w_s = space.newunicode(s) + w_res = ucd.normalize(space, NF_code, w_s) + return space.unwrap(w_res) + return normalize + +all_forms = ['NFC', 'NFD', 'NFKC', 'NFKD'] + +# For every (n1, n2, n3) triple, applying n1 then n2 must be the same +# as applying n3. +# Reference: http://unicode.org/reports/tr15/#Design_Goals +compositions = [ + ('NFC', 'NFC', 'NFC'), + ('NFC', 'NFD', 'NFD'), + ('NFC', 'NFKC', 'NFKC'), + ('NFC', 'NFKD', 'NFKD'), + ('NFD', 'NFC', 'NFC'), + ('NFD', 'NFD', 'NFD'), + ('NFD', 'NFKC', 'NFKC'), + ('NFD', 'NFKD', 'NFKD'), + ('NFKC', 'NFC', 'NFKC'), + ('NFKC', 'NFD', 'NFKD'), + ('NFKC', 'NFKC', 'NFKC'), + ('NFKC', 'NFKD', 'NFKD'), + ('NFKD', 'NFC', 'NFKC'), + ('NFKD', 'NFD', 'NFKD'), + ('NFKD', 'NFKC', 'NFKC'), + ('NFKD', 'NFKD', 'NFKD'), +] + + +@pytest.mark.parametrize('NF1, NF2, NF3', compositions) +@example(s=u'---\uafb8\u11a7---') # issue 2289 +@settings(max_examples=1000) +@given(s=st.text()) +def test_composition(s, space, NF1, NF2, NF3): + norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] + assert norm2(norm1(s)) == norm3(s) diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py --- a/rpython/jit/codewriter/jitcode.py +++ b/rpython/jit/codewriter/jitcode.py @@ -146,14 +146,13 @@ def get_register_index_f(self, index): return ord(self.live_f[index]) - def enumerate_vars(self, callback_i, callback_r, callback_f, spec, index): + def enumerate_vars(self, callback_i, callback_r, callback_f, spec): for i in range(self.get_register_count_i()): - index = callback_i(index, self.get_register_index_i(i)) + callback_i(self.get_register_index_i(i)) for i in range(self.get_register_count_r()): - index = callback_r(index, self.get_register_index_r(i)) + callback_r(self.get_register_index_r(i)) for i in range(self.get_register_count_f()): - index = callback_f(index, self.get_register_index_f(i)) - return index + callback_f(self.get_register_index_f(i)) enumerate_vars._annspecialcase_ = 'specialize:arg(4)' _liveness_cache = {} diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -85,13 +85,14 @@ """ This represents ops() with a jump at the end that goes to some loop, we need to deal with virtual state and inlining of short preamble """ - def __init__(self, trace, runtime_boxes, call_pure_results=None, + def __init__(self, trace, runtime_boxes, resumestorage=None, call_pure_results=None, enable_opts=None, inline_short_preamble=False): self.trace = trace self.runtime_boxes = runtime_boxes self.call_pure_results = call_pure_results self.enable_opts = enable_opts self.inline_short_preamble = inline_short_preamble + self.resumestorage = resumestorage def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer @@ -100,7 +101,8 @@ return opt.optimize_bridge(self.trace, self.runtime_boxes, self.call_pure_results, self.inline_short_preamble, - self.box_names_memo) + self.box_names_memo, + self.resumestorage) class UnrolledLoopData(CompileData): """ This represents label() ops jump with extra info that's from the @@ -870,10 +872,9 @@ class ResumeGuardDescr(AbstractResumeGuardDescr): - _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals', + _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals', 'rd_pendingfields', 'status') rd_numb = lltype.nullptr(NUMBERING) - rd_count = 0 rd_consts = None rd_virtuals = None rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) @@ -882,7 +883,6 @@ if isinstance(other, ResumeGuardCopiedDescr): other = other.prev assert isinstance(other, ResumeGuardDescr) - self.rd_count = other.rd_count self.rd_consts = other.rd_consts self.rd_pendingfields = other.rd_pendingfields self.rd_virtuals = other.rd_virtuals @@ -895,7 +895,6 @@ def store_final_boxes(self, guard_op, boxes, metainterp_sd): guard_op.setfailargs(boxes) - self.rd_count = len(boxes) self.store_hash(metainterp_sd) def clone(self): @@ -1077,7 +1076,15 @@ call_pure_results = metainterp.call_pure_results if metainterp.history.ends_with_jump: - data = BridgeCompileData(trace, runtime_boxes, + if isinstance(resumekey, ResumeGuardCopiedDescr): + key = resumekey.prev + assert isinstance(key, ResumeGuardDescr) + elif isinstance(resumekey, ResumeFromInterpDescr): + key = None + else: + key = resumekey + assert isinstance(key, ResumeGuardDescr) + data = BridgeCompileData(trace, runtime_boxes, key, call_pure_results=call_pure_results, enable_opts=enable_opts, inline_short_preamble=inline_short_preamble) diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py --- a/rpython/jit/metainterp/opencoder.py +++ b/rpython/jit/metainterp/opencoder.py @@ -66,7 +66,7 @@ assert isinstance(snapshot, TopSnapshot) self.vable_array = snapshot.vable_array self.vref_array = snapshot.vref_array - self.size = len(self.vable_array) + len(self.vref_array) + 2 + self.size = len(self.vable_array) + len(self.vref_array) + 3 jc_index, pc = unpack_uint(snapshot.packed_jitcode_pc) self.framestack = [] if jc_index == 2**16-1: diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py new file mode 100644 --- /dev/null +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -0,0 +1,135 @@ +""" Code to feed information from the optimizer via the resume code into the +optimizer of the bridge attached to a guard. """ + +from rpython.jit.metainterp import resumecode + + +# adds the following sections at the end of the resume code: +# +# ---- known classes +# <bitfield> size is the number of reference boxes in the liveboxes +# 1 klass known +# 0 klass unknown +# (the class is found by actually looking at the runtime value) +# the bits are bunched in bunches of 7 +# +# ---- heap knowledge +# <length> +# (<box1> <descr> <box2>) length times, if getfield(box1, descr) == box2 +# both boxes should be in the liveboxes +# +# ---- + + +# maybe should be delegated to the optimization classes? + +def tag_box(box, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import Const + if isinstance(box, Const): + return memo.getconst(box) + else: + return liveboxes_from_env[box] # has to exist + +def decode_box(resumestorage, tagged, liveboxes, cpu): + from rpython.jit.metainterp.resume import untag, TAGCONST, TAGINT, TAGBOX + from rpython.jit.metainterp.resume import NULLREF, TAG_CONST_OFFSET, tagged_eq + from rpython.jit.metainterp.history import ConstInt + num, tag = untag(tagged) + # NB: the TAGVIRTUAL case can't happen here, because this code runs after + # virtuals are already forced again + if tag == TAGCONST: + if tagged_eq(tagged, NULLREF): + box = cpu.ts.CONST_NULL + else: + box = resumestorage.rd_consts[num - TAG_CONST_OFFSET] + elif tag == TAGINT: + box = ConstInt(num) + elif tag == TAGBOX: + box = liveboxes[num] + else: + raise AssertionError("unreachable") + return box + +def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): + available_boxes = {} + for box in liveboxes: + if box is not None and box in liveboxes_from_env: + available_boxes[box] = None + metainterp_sd = optimizer.metainterp_sd + + # class knowledge is stored as bits, true meaning the class is known, false + # means unknown. on deserializing we look at the bits, and read the runtime + # class for the known classes (which has to be the same in the bridge) and + # mark that as known. this works for guard_class too: the class is only + # known *after* the guard + bitfield = 0 + shifts = 0 + for box in liveboxes: + if box is None or box.type != "r": + continue + info = optimizer.getptrinfo(box) + known_class = info is not None and info.get_known_class(optimizer.cpu) is not None + bitfield <<= 1 + bitfield |= known_class + shifts += 1 + if shifts == 6: + numb_state.append_int(bitfield) + bitfield = shifts = 0 + if shifts: + numb_state.append_int(bitfield << (6 - shifts)) + + # heap knowledge: we store triples of known heap fields in non-virtual + # structs + # XXX could be extended to arrays + if optimizer.optheap: + triples = optimizer.optheap.serialize_optheap(available_boxes) + # can only encode descrs that have a known index into + # metainterp_sd.all_descrs + triples = [triple for triple in triples if triple[1].descr_index != -1] + numb_state.append_int(len(triples)) + for box1, descr, box2 in triples: + index = descr.descr_index + numb_state.append_short(tag_box(box1, liveboxes_from_env, memo)) + numb_state.append_int(index) + numb_state.append_short(tag_box(box2, liveboxes_from_env, memo)) + else: + numb_state.append_int(0) + +def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + reader = resumecode.Reader(resumestorage.rd_numb) + assert len(frontend_boxes) == len(liveboxes) + metainterp_sd = optimizer.metainterp_sd + + # skip resume section + startcount = reader.next_item() + reader.jump(startcount - 1) + + # class knowledge + bitfield = 0 + mask = 0 + for i, box in enumerate(liveboxes): + if box.type != "r": + continue + if not mask: + bitfield = reader.next_item() + mask = 0b100000 + class_known = bitfield & mask + mask >>= 1 + if class_known: + cls = optimizer.cpu.ts.cls_of_box(frontend_boxes[i]) + optimizer.make_constant_class(box, cls) + + # heap knowledge + if not optimizer.optheap: + return + length = reader.next_item() + result = [] + for i in range(length): + tagged = reader.next_item() + box1 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) + index = reader.next_item() + descr = metainterp_sd.all_descrs[index] + tagged = reader.next_item() + box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) + result.append((box1, descr, box2)) + optimizer.optheap.deserialize_optheap(result) diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -48,6 +48,7 @@ # that has a non-None entry at # info._fields[descr.get_index()] # must be in cache_infos + assert structop.type == 'r' self.cached_structs.append(structop) self.cached_infos.append(info) @@ -693,6 +694,36 @@ self._seen_guard_not_invalidated = True return self.emit(op) + def serialize_optheap(self, available_boxes): + result = [] + for descr, cf in self.cached_fields.iteritems(): + if cf._lazy_set: + continue # XXX safe default for now + parent_descr = descr.get_parent_descr() + if not parent_descr.is_object(): + continue # XXX could be extended to non-instance objects + for i, box1 in enumerate(cf.cached_structs): + if box1 not in available_boxes: + continue + structinfo = cf.cached_infos[i] + box2 = structinfo.getfield(descr).get_box_replacement() + if isinstance(box2, Const) or box2 in available_boxes: + result.append((box1, descr, box2)) + return result + + def deserialize_optheap(self, triples): + for box1, descr, box2 in triples: + parent_descr = descr.get_parent_descr() + assert parent_descr.is_object() + structinfo = box1.get_forwarded() + if not isinstance(structinfo, info.AbstractVirtualPtrInfo): + structinfo = info.InstancePtrInfo(parent_descr) + structinfo.init_fields(parent_descr, descr.get_index()) + box1.set_forwarded(structinfo) + + cf = self.field_cache(descr) + structinfo.setfield(descr, box1, box2, optheap=self, cf=cf) + dispatch_opt = make_dispatcher_method(OptHeap, 'optimize_', default=OptHeap.emit) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -70,7 +70,7 @@ def same_info(self, other): return self is other - def getstrlen(self, op, string_optimizer, mode, create_ops=True): + def getstrlen(self, op, string_optimizer, mode): return None def getstrhash(self, op, mode): @@ -777,21 +777,20 @@ # XXX we can do better if we know it's an array return IntLowerBound(0) else: - return ConstIntBound(self.getstrlen(None, None, mode).getint()) - - def getstrlen(self, op, string_optimizer, mode, create_ops=True): + return ConstIntBound(self.getstrlen1(mode)) + + def getstrlen(self, op, string_optimizer, mode): + return ConstInt(self.getstrlen1(mode)) + + def getstrlen1(self, mode): from rpython.jit.metainterp.optimizeopt import vstring if mode is vstring.mode_string: s = self._unpack_str(vstring.mode_string) - if s is None: - return None - return ConstInt(len(s)) + return len(s) else: s = self._unpack_str(vstring.mode_unicode) - if s is None: - return None - return ConstInt(len(s)) + return len(s) def getstrhash(self, op, mode): from rpython.jit.metainterp.optimizeopt import vstring @@ -812,7 +811,7 @@ from rpython.jit.metainterp.optimizeopt import vstring from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0 - lgt = self.getstrlen(op, string_optimizer, mode, False) + lgt = self.getstrlen(op, string_optimizer, mode) return vstring.copy_str_content(string_optimizer, self._const, targetbox, CONST_0, offsetbox, lgt, mode) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -297,6 +297,7 @@ self.optrewrite = None self.optearlyforce = None self.optunroll = None + self._really_emitted_operation = None self._last_guard_op = None diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py @@ -30,6 +30,7 @@ self.add_guard_future_condition(bridge) trace = oparser.convert_loop_to_trace(bridge, FakeMetaInterpStaticData(self.cpu)) data = compile.BridgeCompileData(trace, self.convert_values(bridge.operations[-1].getarglist(), bridge_values), + None, enable_opts=self.enable_opts, inline_short_preamble=inline_short_preamble) bridge_info, ops = self._do_optimize_loop(data) diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -232,9 +232,15 @@ return label_vs def optimize_bridge(self, trace, runtime_boxes, call_pure_results, - inline_short_preamble, box_names_memo): + inline_short_preamble, box_names_memo, resumestorage): + from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge + frontend_inputargs = trace.inputargs trace = trace.get_iter() self._check_no_forwarding([trace.inputargs]) + if resumestorage: + deserialize_optimizer_knowledge(self.optimizer, + resumestorage, frontend_inputargs, + trace.inputargs) info, ops = self.optimizer.propagate_all_forward(trace, call_pure_results, False) jump_op = info.jump_op diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -752,7 +752,6 @@ def __init__(self, r=lltype.nullptr(llmemory.GCREF.TO)): self.setref_base(r) - self.datatype = 'r' def reset_value(self): self.setref_base(lltype.nullptr(llmemory.GCREF.TO)) diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -162,25 +162,13 @@ UNINITIALIZED = tag(-2, TAGCONST) # used for uninitialized string characters TAG_CONST_OFFSET = 0 -class NumberingState(object): +class NumberingState(resumecode.Writer): def __init__(self, size): + resumecode.Writer.__init__(self, size) self.liveboxes = {} - self.current = [rffi.cast(rffi.SHORT, 0)] * size - self._pos = 0 self.num_boxes = 0 self.num_virtuals = 0 - def append_short(self, item): - self.current[self._pos] = item - self._pos += 1 - - def append_int(self, item): - short = rffi.cast(rffi.SHORT, item) - assert rffi.cast(lltype.Signed, short) == item - return self.append_short(short) - - def create_numbering(self): - return resumecode.create_numbering(self.current) class ResumeDataLoopMemo(object): @@ -268,6 +256,8 @@ def number(self, optimizer, position, trace): snapshot_iter = trace.get_snapshot_iter(position) numb_state = NumberingState(snapshot_iter.size) + numb_state.append_int(0) # patch later: size of resume section + numb_state.append_int(0) # patch later: number of failargs arr = snapshot_iter.vable_array @@ -287,6 +277,7 @@ numb_state.append_int(pc) self._number_boxes( snapshot_iter, snapshot.box_array, optimizer, numb_state) + numb_state.patch_current_size(0) return numb_state @@ -471,6 +462,9 @@ self._number_virtuals(liveboxes, optimizer, num_virtuals) self._add_pending_fields(optimizer, pending_setfields) + numb_state.patch(1, len(liveboxes)) + + self._add_optimizer_sections(numb_state, liveboxes, liveboxes_from_env) storage.rd_numb = numb_state.create_numbering() storage.rd_consts = self.memo.consts return liveboxes[:] @@ -590,6 +584,12 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] + def _add_optimizer_sections(self, numb_state, liveboxes, liveboxes_from_env): + # add extra information about things the optimizer learned + from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge + serialize_optimizer_knowledge( + self.optimizer, numb_state, liveboxes, liveboxes_from_env, self.memo) + class AbstractVirtualInfo(object): kind = REF is_about_raw = False @@ -931,9 +931,10 @@ def _init(self, cpu, storage): self.cpu = cpu - self.numb = storage.rd_numb - self.cur_index = 0 - self.count = storage.rd_count + self.resumecodereader = resumecode.Reader(storage.rd_numb) + items_resume_section = self.resumecodereader.next_item() + self.items_resume_section = items_resume_section + self.count = self.resumecodereader.next_item() self.consts = storage.rd_consts def _prepare(self, storage): @@ -941,14 +942,21 @@ self._prepare_pendingfields(storage.rd_pendingfields) def read_jitcode_pos_pc(self): - jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb, - self.cur_index) - pc, self.cur_index = resumecode.numb_next_item(self.numb, - self.cur_index) + jitcode_pos = self.resumecodereader.next_item() + pc = self.resumecodereader.next_item() return jitcode_pos, pc + def next_int(self): + return self.decode_int(self.resumecodereader.next_item()) + + def next_ref(self): + return self.decode_ref(self.resumecodereader.next_item()) + + def next_float(self): + return self.decode_float(self.resumecodereader.next_item()) + def done_reading(self): - return self.cur_index >= len(self.numb.code) + return self.resumecodereader.items_read >= self.items_resume_section def getvirtual_ptr(self, index): # Returns the index'th virtual, building it lazily if needed. @@ -1025,29 +1033,22 @@ def _prepare_next_section(self, info): # Use info.enumerate_vars(), normally dispatching to # rpython.jit.codewriter.jitcode. Some tests give a different 'info'. - self.cur_index = info.enumerate_vars(self._callback_i, - self._callback_r, - self._callback_f, - self.unique_id, # <-- annotation hack - self.cur_index) + info.enumerate_vars(self._callback_i, + self._callback_r, + self._callback_f, + self.unique_id) # <-- annotation hack - def _callback_i(self, index, register_index): - item, index = resumecode.numb_next_item(self.numb, index) - value = self.decode_int(item) + def _callback_i(self, register_index): + value = self.next_int() self.write_an_int(register_index, value) - return index - def _callback_r(self, index, register_index): - item, index = resumecode.numb_next_item(self.numb, index) - value = self.decode_ref(item) + def _callback_r(self, register_index): + value = self.next_ref() self.write_a_ref(register_index, value) - return index - def _callback_f(self, index, register_index): - item, index = resumecode.numb_next_item(self.numb, index) - value = self.decode_float(item) + def _callback_f(self, register_index): + value = self.next_float() self.write_a_float(register_index, value) - return index # ---------- when resuming for pyjitpl.py, make boxes ---------- @@ -1057,6 +1058,7 @@ boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info, greenfield_info) virtualizable_boxes, virtualref_boxes = boxes + while not resumereader.done_reading(): jitcode_pos, pc = resumereader.read_jitcode_pos_pc() jitcode = metainterp.staticdata.jitcodes[jitcode_pos] @@ -1076,7 +1078,7 @@ self._init(metainterp.cpu, storage) self.deadframe = deadframe self.metainterp = metainterp - self.liveboxes = [None] * storage.rd_count + self.liveboxes = [None] * self.count self._prepare(storage) def consume_boxes(self, info, boxes_i, boxes_r, boxes_f): @@ -1085,42 +1087,30 @@ self.boxes_f = boxes_f self._prepare_next_section(info) - def consume_virtualizable_boxes(self, vinfo, index): + def consume_virtualizable_boxes(self, vinfo): # we have to ignore the initial part of 'nums' (containing vrefs), # find the virtualizable from nums[-1], and use it to know how many # boxes of which type we have to return. This does not write # anything into the virtualizable. - numb = self.numb - item, index = resumecode.numb_next_item(numb, index) - virtualizablebox = self.decode_ref(item) + virtualizablebox = self.next_ref() virtualizable = vinfo.unwrap_virtualizable_box(virtualizablebox) - return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox, - numb, index) + return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox) - def consume_virtualref_boxes(self, index): + def consume_virtualref_boxes(self): # Returns a list of boxes, assumed to be all BoxPtrs. # We leave up to the caller to call vrefinfo.continue_tracing(). - size, index = resumecode.numb_next_item(self.numb, index) - if size == 0: - return [], index - lst = [] - for i in range(size * 2): - item, index = resumecode.numb_next_item(self.numb, index) - lst.append(self.decode_ref(item)) - return lst, index + size = self.resumecodereader.next_item() + return [self.next_ref() for i in range(size * 2)] def consume_vref_and_vable_boxes(self, vinfo, ginfo): - vable_size, index = resumecode.numb_next_item(self.numb, 0) + vable_size = self.resumecodereader.next_item() if vinfo is not None: - virtualizable_boxes, index = self.consume_virtualizable_boxes(vinfo, - index) + virtualizable_boxes = self.consume_virtualizable_boxes(vinfo) elif ginfo is not None: - item, index = resumecode.numb_next_item(self.numb, index) - virtualizable_boxes = [self.decode_ref(item)] + virtualizable_boxes = [self.next_ref()] else: virtualizable_boxes = None - virtualref_boxes, index = self.consume_virtualref_boxes(index) - self.cur_index = index + virtualref_boxes = self.consume_virtualref_boxes() return virtualizable_boxes, virtualref_boxes def allocate_with_vtable(self, descr=None): @@ -1297,7 +1287,7 @@ self.liveboxes[num] = box return box - def decode_box_of_type(self, TYPE, tagged): + def next_box_of_type(self, TYPE): kind = getkind(TYPE) if kind == 'int': kind = INT @@ -1307,8 +1297,8 @@ kind = FLOAT else: raise AssertionError(kind) - return self.decode_box(tagged, kind) - decode_box_of_type._annspecialcase_ = 'specialize:arg(1)' + return self.decode_box(self.resumecodereader.next_item(), kind) + next_box_of_type._annspecialcase_ = 'specialize:arg(1)' def write_an_int(self, index, box): self.boxes_i[index] = box @@ -1397,64 +1387,54 @@ info = blackholeinterp.get_current_position_info() self._prepare_next_section(info) - def consume_virtualref_info(self, vrefinfo, index): + def consume_virtualref_info(self, vrefinfo): # we have to decode a list of references containing pairs # [..., virtual, vref, ...] and returns the index at the end - size, index = resumecode.numb_next_item(self.numb, index) + size = self.resumecodereader.next_item() if vrefinfo is None or size == 0: assert size == 0 - return index + return for i in range(size): - virtual_item, index = resumecode.numb_next_item( - self.numb, index) - vref_item, index = resumecode.numb_next_item( - self.numb, index) - virtual = self.decode_ref(virtual_item) - vref = self.decode_ref(vref_item) + virtual = self.next_ref() + vref = self.next_ref() # For each pair, we store the virtual inside the vref. vrefinfo.continue_tracing(vref, virtual) - return index - def consume_vable_info(self, vinfo, index): + def consume_vable_info(self, vinfo): # we have to ignore the initial part of 'nums' (containing vrefs), # find the virtualizable from nums[-1], load all other values # from the CPU stack, and copy them into the virtualizable - numb = self.numb - item, index = resumecode.numb_next_item(self.numb, index) - virtualizable = self.decode_ref(item) + virtualizable = self.next_ref() # just reset the token, we'll force it later vinfo.reset_token_gcref(virtualizable) - index = vinfo.write_from_resume_data_partial(virtualizable, self, - index, numb) - return index + vinfo.write_from_resume_data_partial(virtualizable, self) - def load_value_of_type(self, TYPE, tagged): + def load_next_value_of_type(self, TYPE): from rpython.jit.metainterp.warmstate import specialize_value kind = getkind(TYPE) if kind == 'int': - x = self.decode_int(tagged) + x = self.next_int() elif kind == 'ref': - x = self.decode_ref(tagged) + x = self.next_ref() elif kind == 'float': - x = self.decode_float(tagged) + x = self.next_float() else: raise AssertionError(kind) return specialize_value(TYPE, x) - load_value_of_type._annspecialcase_ = 'specialize:arg(1)' + load_next_value_of_type._annspecialcase_ = 'specialize:arg(1)' def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo): - vable_size, index = resumecode.numb_next_item(self.numb, 0) + vable_size = self.resumecodereader.next_item() if self.resume_after_guard_not_forced != 2: if vinfo is not None: - index = self.consume_vable_info(vinfo, index) + self.consume_vable_info(vinfo) if ginfo is not None: - _, index = resumecode.numb_next_item(self.numb, index) - index = self.consume_virtualref_info(vrefinfo, index) + _ = self.resumecodereader.next_item() + self.consume_virtualref_info(vrefinfo) else: - index = resumecode.numb_next_n_items(self.numb, vable_size, index) - vref_size, index = resumecode.numb_next_item(self.numb, index) - index = resumecode.numb_next_n_items(self.numb, vref_size * 2, index) - self.cur_index = index + self.resumecodereader.jump(vable_size) + vref_size = self.resumecodereader.next_item() + self.resumecodereader.jump(vref_size * 2) def allocate_with_vtable(self, descr=None): from rpython.jit.metainterp.executor import exec_new_with_vtable diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py --- a/rpython/jit/metainterp/resumecode.py +++ b/rpython/jit/metainterp/resumecode.py @@ -1,6 +1,9 @@ """ Resume bytecode. It goes as following: + # ----- resume section + [total size of resume section] + [number of failargs] [<length> <virtualizable object> <numb> <numb> <numb>] if vinfo is not None -OR- [1 <ginfo object>] if ginfo is not None @@ -13,10 +16,14 @@ [<pc> <jitcode> <numb> <numb>] ... - until the length of the array. + until the size of the resume section + + # ----- optimization section + <more code> further sections according to bridgeopt.py """ from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib import objectmodel NUMBERINGP = lltype.Ptr(lltype.GcForwardReference()) NUMBERING = lltype.GcStruct('Numbering', @@ -24,33 +31,24 @@ NUMBERINGP.TO.become(NUMBERING) NULL_NUMBER = lltype.nullptr(NUMBERING) -def create_numbering(lst, total=-1): - if total == -1: - total = len(lst) - result = [] - for i in range(total): - item = lst[i] - item = rffi.cast(lltype.Signed, item) - item *= 2 - if item < 0: - item = -1 - item +def append_numbering(lst, item): + item = rffi.cast(lltype.Signed, item) + item *= 2 + if item < 0: + item = -1 - item - assert item >= 0 - if item < 2**7: - result.append(rffi.cast(rffi.UCHAR, item)) - elif item < 2**14: - result.append(rffi.cast(rffi.UCHAR, item | 0x80)) - result.append(rffi.cast(rffi.UCHAR, item >> 7)) - else: - assert item < 2**16 - result.append(rffi.cast(rffi.UCHAR, item | 0x80)) - result.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80)) - result.append(rffi.cast(rffi.UCHAR, item >> 14)) + assert item >= 0 + if item < 2**7: + lst.append(rffi.cast(rffi.UCHAR, item)) + elif item < 2**14: + lst.append(rffi.cast(rffi.UCHAR, item | 0x80)) + lst.append(rffi.cast(rffi.UCHAR, item >> 7)) + else: + assert item < 2**16 + lst.append(rffi.cast(rffi.UCHAR, item | 0x80)) + lst.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80)) + lst.append(rffi.cast(rffi.UCHAR, item >> 14)) - numb = lltype.malloc(NUMBERING, len(result)) - for i in range(len(result)): - numb.code[i] = result[i] - return numb def numb_next_item(numb, index): value = rffi.cast(lltype.Signed, numb.code[index]) @@ -81,3 +79,64 @@ next, i = numb_next_item(numb, i) l.append(next) return l + +class Writer(object): + def __init__(self, size=0): + self.current = objectmodel.newlist_hint(size) + + def append_short(self, item): + self.current.append(item) + + def append_int(self, item): + short = rffi.cast(rffi.SHORT, item) + assert rffi.cast(lltype.Signed, short) == item + return self.append_short(short) + + def create_numbering(self): + final = objectmodel.newlist_hint(len(self.current) * 3) + for item in self.current: + append_numbering(final, item) + numb = lltype.malloc(NUMBERING, len(final)) + for i, elt in enumerate(final): + numb.code[i] = elt + return numb + + def patch_current_size(self, index): + self.patch(index, len(self.current)) + + def patch(self, index, item): + self.current[index] = item + +def create_numbering(l): + w = Writer() + for item in l: + w.append_int(item) + return w.create_numbering() + + +class Reader(object): + def __init__(self, code): + self.code = code + self.cur_pos = 0 # index into the code + self.items_read = 0 # number of items read + + def next_item(self): + result, self.cur_pos = numb_next_item(self.code, self.cur_pos) + self.items_read += 1 + return result + + def peek(self): + result, _ = numb_next_item(self.code, self.cur_pos) + return result + + def jump(self, size): + """ jump n items forward without returning anything """ + index = self.cur_pos + for i in range(size): + _, index = numb_next_item(self.code, index) + self.items_read += size + self.cur_pos = index + + def unpack(self): + # mainly for debugging + return unpack_numbering(self.code) diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py new file mode 100644 --- /dev/null +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -0,0 +1,188 @@ +# tests that check that information is fed from the optimizer into the bridges + +import math +from rpython.rlib import jit +from rpython.jit.metainterp.test.support import LLJitMixin +from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge +from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge +from rpython.jit.metainterp.resoperation import InputArgRef, InputArgInt +from rpython.jit.metainterp.resume import NumberingState +from rpython.jit.metainterp.resumecode import unpack_numbering +from rpython.jit.metainterp.optimizeopt.info import InstancePtrInfo + +from hypothesis import strategies, given + +class FakeTS(object): + def __init__(self, dct): + self.dct = dct + + def cls_of_box(self, box): + return self.dct[box] + + +class FakeCPU(object): + def __init__(self, dct): + self.ts = FakeTS(dct) + +class FakeOptimizer(object): + metainterp_sd = None + optheap = None + + def __init__(self, dct={}, cpu=None): + self.dct = dct + self.constant_classes = {} + self.cpu = cpu + + def getptrinfo(self, arg): + return self.dct.get(arg, None) + + def make_constant_class(self, arg, cls): + self.constant_classes[arg] = cls + +class FakeClass(object): + pass + +class FakeStorage(object): + def __init__(self, numb): + self.rd_numb = numb + +def test_known_classes(): + box1 = InputArgRef() + box2 = InputArgRef() + box3 = InputArgRef() + + cls = FakeClass() + dct = {box1: InstancePtrInfo(known_class=cls)} + optimizer = FakeOptimizer(dct) + + numb_state = NumberingState(4) + numb_state.append_int(1) # size of resume block + liveboxes = [InputArgInt(), box2, box1, box3] + + serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) + + assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0] + + rbox1 = InputArgRef() + rbox2 = InputArgRef() + rbox3 = InputArgRef() + after_optimizer = FakeOptimizer(cpu=FakeCPU({rbox1: cls})) + deserialize_optimizer_knowledge( + after_optimizer, FakeStorage(numb_state.create_numbering()), + [InputArgInt(), rbox2, rbox1, rbox3], liveboxes) + assert box1 in after_optimizer.constant_classes + assert box2 not in after_optimizer.constant_classes + assert box3 not in after_optimizer.constant_classes + + +box_strategy = strategies.builds(InputArgInt) | strategies.builds(InputArgRef) +tuples = strategies.tuples(box_strategy, strategies.booleans()).filter( + lambda (box, known_class): isinstance(box, InputArgRef) or not known_class) +boxes_known_classes = strategies.lists(tuples, min_size=1) + +@given(boxes_known_classes) +def test_random_class_knowledge(boxes_known_classes): + cls = FakeClass() + dct1 = {box: InstancePtrInfo(known_class=cls) + for box, known_class in boxes_known_classes + if known_class} + optimizer = FakeOptimizer(dct1) + + refboxes = [box for (box, _) in boxes_known_classes + if isinstance(box, InputArgRef)] + + numb_state = NumberingState(1) + numb_state.append_int(1) # size of resume block + liveboxes = [box for (box, _) in boxes_known_classes] + + serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) + + assert len(numb_state.create_numbering().code) == 2 + math.ceil(len(refboxes) / 6.0) + + dct = {box: cls + for box, known_class in boxes_known_classes + if known_class} + after_optimizer = FakeOptimizer(cpu=FakeCPU(dct)) + deserialize_optimizer_knowledge( + after_optimizer, FakeStorage(numb_state.create_numbering()), + liveboxes, liveboxes) + for box, known_class in boxes_known_classes: + assert (box in after_optimizer.constant_classes) == known_class + +class TestOptBridge(LLJitMixin): + # integration tests + def test_bridge_guard_class(self): + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a']) + class A(object): + def f(self): + return 1 + class B(A): + def f(self): + return 2 + def f(x, y, n): + if x: + a = A() + else: + a = B() + a.x = 0 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a) + res += a.f() + a.x += 1 + if y > n: + res += 1 + res += a.f() + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + assert res == f(6, 32, 16) + self.check_trace_count(3) + self.check_resops(guard_class=1) + + def test_bridge_field_read(self): + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a']) + class A(object): + def f(self): + return 1 + class B(A): + def f(self): + return 2 + class M(object): + _immutable_fields_ = ['x'] + def __init__(self, x): + self.x = x + + m1 = M(1) + m2 = M(2) + def f(x, y, n): + if x: + a = A() + a.m = m1 + a.n = n + else: + a = B() + a.m = m2 + a.n = n + a.x = 0 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a) + n1 = a.n + m = jit.promote(a.m) + res += m.x + a.x += 1 + if y > n: + res += 1 + m = jit.promote(a.m) + res += m.x + res += n1 + a.n + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + assert res == f(6, 32, 16) + self.check_trace_count(3) + self.check_resops(guard_value=1) + self.check_resops(getfield_gc_i=4) # 3x a.x, 1x a.n + self.check_resops(getfield_gc_r=1) # in main loop + diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py --- a/rpython/jit/metainterp/test/test_compile.py +++ b/rpython/jit/metainterp/test/test_compile.py @@ -79,6 +79,7 @@ def test_compile_loop(): cpu = FakeCPU() staticdata = FakeMetaInterpStaticData() + staticdata.all_descrs = LLtypeMixin.cpu.setup_descrs() staticdata.cpu = cpu staticdata.jitlog = jl.JitLogger(cpu) staticdata.jitlog.trace_id = 1 diff --git a/rpython/jit/metainterp/test/test_greenfield.py b/rpython/jit/metainterp/test/test_greenfield.py --- a/rpython/jit/metainterp/test/test_greenfield.py +++ b/rpython/jit/metainterp/test/test_greenfield.py @@ -49,7 +49,7 @@ # res = self.meta_interp(g, [7]) assert res == -22 - self.check_trace_count(6) + self.check_trace_count(4) self.check_resops(guard_value=0) def test_green_field_3(self): diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -36,10 +36,12 @@ rd_consts = [] rd_virtuals = None rd_pendingfields = None - rd_count = 0 class FakeOptimizer(object): + metainterp_sd = None + optheap = None + def __init__(self, trace=None): self.trace = trace @@ -251,18 +253,17 @@ def get_current_position_info(self): class MyInfo: @staticmethod - def enumerate_vars(callback_i, callback_r, callback_f, _, index): + def enumerate_vars(callback_i, callback_r, callback_f, _): count_i = count_r = count_f = 0 for ARG in self.ARGS: if ARG == lltype.Signed: - index = callback_i(index, count_i); count_i += 1 + callback_i(count_i); count_i += 1 elif ARG == llmemory.GCREF: - index = callback_r(index, count_r); count_r += 1 + callback_r(count_r); count_r += 1 elif ARG == longlong.FLOATSTORAGE: - index = callback_f(index, count_f); count_f += 1 + callback_f(count_f); count_f += 1 else: assert 0 - return index return MyInfo() def setarg_i(self, index, value): @@ -289,7 +290,8 @@ assert bh.written_f == expected_f -Numbering = create_numbering +def Numbering(l): + return create_numbering([len(l)] + l) # prefix index to the end of thing def tagconst(i): return tag(i + TAG_CONST_OFFSET, TAGCONST) @@ -299,12 +301,11 @@ c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)] storage = Storage() storage.rd_consts = [c1, c2, c3] - numb = Numbering([tag(0, TAGBOX), tagconst(0), + numb = Numbering([3, tag(0, TAGBOX), tagconst(0), NULLREF, tag(0, TAGBOX), tag(1, TAGBOX)] + - [tagconst(1), tagconst(2)] + + [tagconst(1), tagconst(2)] + [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)]) storage.rd_numb = numb - storage.rd_count = 3 # cpu = MyCPU([42, gcref1, -66]) metainterp = MyMetaInterp(cpu) @@ -345,7 +346,7 @@ def test_simple_read_tagged_ints(): storage = Storage() storage.rd_consts = [] - numb = Numbering([tag(100, TAGINT)]) + numb = Numbering([1, tag(100, TAGINT)]) storage.rd_numb = numb # cpu = MyCPU([]) @@ -362,10 +363,9 @@ return s class FakeStorage(object): rd_virtuals = [FakeVinfo(), None] - rd_numb = [] + rd_numb = Numbering([1]) rd_consts = [] rd_pendingfields = None - rd_count = 0 class FakeMetainterp(object): _already_allocated_resume_virtuals = None cpu = None @@ -773,12 +773,12 @@ assert untag(tagged) == (44, TAGINT) tagged = memo.getconst(ConstInt(-3)) assert untag(tagged) == (-3, TAGINT) - const = ConstInt(50000) + const = ConstInt(5000000) tagged = memo.getconst(const) index, tagbits = untag(tagged) assert tagbits == TAGCONST assert memo.consts[index - TAG_CONST_OFFSET] is const - tagged = memo.getconst(ConstInt(50000)) + tagged = memo.getconst(ConstInt(5000000)) index2, tagbits = untag(tagged) assert tagbits == TAGCONST assert index2 == index @@ -858,7 +858,7 @@ base = [0, 0, tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), tag(0, TAGBOX), tag(2, TAGINT)] - assert unpack_numbering(numb) == [0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX), + assert unpack_numbering(numb) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(1, TAGINT)] t.append(0) snap2 = t.create_top_snapshot(FakeJitCode("jitcode", 0), 2, Frame(env2), @@ -872,7 +872,7 @@ assert numb_state2.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} assert numb_state2.liveboxes is not numb_state.liveboxes - assert unpack_numbering(numb2) == [0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX), + assert unpack_numbering(numb2) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(3, TAGINT)] t.append(0) @@ -894,7 +894,7 @@ assert numb_state3.num_virtuals == 0 assert numb_state3.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)} - assert unpack_numbering(numb3) == ([0, 2, tag(3, TAGINT), tag(4, TAGINT), + assert unpack_numbering(numb3) == ([17, 0, 0, 2, tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), tag(3, TAGINT)] + base + [0, 2]) @@ -911,7 +911,7 @@ assert numb_state4.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL)} - assert unpack_numbering(numb4) == [0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL), + assert unpack_numbering(numb4) == [17, 0, 0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3, TAGINT)] + base + [0, 2] t.append(0) @@ -930,7 +930,7 @@ assert numb_state5.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)} - assert unpack_numbering(numb5) == [ + assert unpack_numbering(numb5) == [22, 0, 3, tag(0, TAGBOX), tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL), 0] + base + [ 2, 1, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3, TAGINT) @@ -949,15 +949,17 @@ numb_state = memo.number(FakeOptimizer(), 0, i) numb = numb_state.create_numbering() l = unpack_numbering(numb) - assert l[0] == 0 + assert l[0] == len(l) + assert l[1] == 0 assert l[1] == 0 assert l[2] == 0 assert l[3] == 0 + assert l[4] == 0 mapping = dict(zip(inpargs, i.inputargs)) for i, item in enumerate(lst): - v, tag = untag(l[i + 4]) + v, tag = untag(l[i + 6]) if tag == TAGBOX: - assert l[i + 4] == numb_state.liveboxes[mapping[item]] + assert l[i + 6] == numb_state.liveboxes[mapping[item]] elif tag == TAGCONST: assert memo.consts[v].getint() == item.getint() elif tag == TAGINT: @@ -1069,15 +1071,15 @@ cpu = MyCPU([]) reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage, "deadframe") reader.consume_vref_and_vable(None, None, None) - reader.cur_index += 2 # framestack + reader.resumecodereader.jump(2) # framestack _next_section(reader, sys.maxint, 1, sys.maxint, 2**16) - reader.cur_index += 2 # framestack + reader.resumecodereader.jump(2) # framestack _next_section(reader, 2, 3) - reader.cur_index += 2 # framestack + reader.resumecodereader.jump(2) # framestack _next_section(reader, sys.maxint, 2**16, -65) def test_virtual_adder_memo_const_sharing(): - b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] + b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**23), ConstInt(-65)] storage, t = make_storage(b1s, b2s, b3s) metainterp_sd = FakeMetaInterpStaticData() memo = ResumeDataLoopMemo(metainterp_sd) @@ -1087,7 +1089,7 @@ assert len(memo.consts) == 2 assert storage.rd_consts is memo.consts - b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)] + b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**24), ConstInt(-65)] storage2, t = make_storage(b1s, b2s, b3s) i = t.get_iter() modifier2 = ResumeDataVirtualAdder(FakeOptimizer(i), storage2, storage2, @@ -1112,9 +1114,10 @@ return True class MyInfo: @staticmethod - def enumerate_vars(callback_i, callback_r, callback_f, _, index): - while index < len(self.numb.code): - tagged, _ = resumecode.numb_next_item(self.numb, index) + def enumerate_vars(callback_i, callback_r, callback_f, _): + index = 0 + while not self.done_reading(): + tagged = self.resumecodereader.peek() _, tag = untag(tagged) if tag == TAGVIRTUAL: kind = REF @@ -1122,20 +1125,21 @@ kind = Whatever() box = self.decode_box(tagged, kind) if box.type == INT: - index = callback_i(index, index) + callback_i(index) elif box.type == REF: - index = callback_r(index, index) + callback_r(index) elif box.type == FLOAT: - index = callback_f(index, index) + callback_f(index) else: assert 0 + index += 1 - size, self.cur_index = resumecode.numb_next_item(self.numb, 0) + size = self.resumecodereader.next_item() assert size == 0 - size, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index) + size = self.resumecodereader.next_item() assert size == 0 - pc, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index) - jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index) + pc = self.resumecodereader.next_item() + jitcode_pos = self.resumecodereader.next_item() self._prepare_next_section(MyInfo()) return self.lst @@ -1228,7 +1232,7 @@ liveboxes = [] modifier._number_virtuals(liveboxes, FakeOptimizer(), 0) storage.rd_consts = memo.consts[:] - storage.rd_numb = None + storage.rd_numb = Numbering([0]) # resume b3t, b5t = [IntFrontendOp(0), RefFrontendOp(0)] b5t.setref_base(demo55o) @@ -1299,7 +1303,7 @@ modifier._number_virtuals(liveboxes, FakeOptimizer(), 0) dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] - storage.rd_numb = None + storage.rd_numb = Numbering([0]) # resume b1t, b3t, b4t = [IntFrontendOp(0), IntFrontendOp(0), IntFrontendOp(0)] b1t.setint(11) @@ -1352,7 +1356,7 @@ modifier._number_virtuals(liveboxes, FakeOptimizer(), 0) dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] - storage.rd_numb = None + storage.rd_numb = Numbering([0]) b4t = RefFrontendOp(0) newboxes = _resume_remap(liveboxes, [#b2s -- virtual b4s], b4t) @@ -1398,7 +1402,7 @@ modifier._add_pending_fields(FakeOptimizer(), [ ResOperation(rop.SETFIELD_GC, [b2s, b4s], descr=LLtypeMixin.nextdescr)]) storage.rd_consts = memo.consts[:] - storage.rd_numb = None + storage.rd_numb = Numbering([0]) # resume demo55.next = lltype.nullptr(LLtypeMixin.NODE) b2t = RefFrontendOp(0) diff --git a/rpython/jit/metainterp/test/test_resumecode.py b/rpython/jit/metainterp/test/test_resumecode.py --- a/rpython/jit/metainterp/test/test_resumecode.py +++ b/rpython/jit/metainterp/test/test_resumecode.py @@ -1,29 +1,62 @@ - -from rpython.jit.metainterp.resumecode import NUMBERING, NULL_NUMBER from rpython.jit.metainterp.resumecode import create_numbering,\ - unpack_numbering + unpack_numbering, Reader, Writer from rpython.rtyper.lltypesystem import lltype -from hypothesis import strategies, given +from hypothesis import strategies, given, example +examples = [ + [1, 2, 3, 4, 257, 10000, 13, 15], + [1, 2, 3, 4], + range(1, 10, 2), + [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000] +] -def test_pack_unpack(): - examples = [ - [1, 2, 3, 4, 257, 10000, 13, 15], - [1, 2, 3, 4], - range(1, 10, 2), - [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000] - ] - for l in examples: - n = create_numbering(l) - assert unpack_numbering(n) == l +def hypothesis_and_examples(func): + func = given(strategies.lists(strategies.integers(-2**15, 2**15-1)))(func) + for ex in examples: + func = example(ex)(func) + return func -@given(strategies.lists(strategies.integers(-2**15, 2**15-1))) +@hypothesis_and_examples def test_roundtrip(l): n = create_numbering(l) assert unpack_numbering(n) == l -@given(strategies.lists(strategies.integers(-2**15, 2**15-1))) +@hypothesis_and_examples def test_compressing(l): n = create_numbering(l) assert len(n.code) <= len(l) * 3 + +@hypothesis_and_examples +def test_reader(l): + n = create_numbering(l) + r = Reader(n) + for i, elt in enumerate(l): + assert r.items_read == i + item = r.next_item() + assert elt == item + +@hypothesis_and_examples +def test_writer(l): + for size in [len(l), 0]: + w = Writer(len(l)) + for num in l: + w.append_int(num) + n = w.create_numbering() + assert unpack_numbering(n) == l + +@hypothesis_and_examples +def test_patch(l): + for middle in range(len(l)): + l1 = l[:middle] + l2 = l[middle:] + w = Writer(len(l)) + w.append_int(0) + for num in l1: + w.append_int(num) + w.patch_current_size(0) + for num in l2: + w.append_int(num) + n = w.create_numbering() + assert unpack_numbering(n)[1:] == l + assert unpack_numbering(n)[0] == middle + 1 diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -2,7 +2,6 @@ from rpython.jit.metainterp import history from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem from rpython.jit.metainterp.warmstate import wrap, unwrap -from rpython.jit.metainterp.resumecode import numb_next_item from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper import rvirtualizable from rpython.rtyper.lltypesystem import lltype, llmemory @@ -127,24 +126,21 @@ size += 1 return size - def write_from_resume_data_partial(virtualizable, reader, index, numb): + def write_from_resume_data_partial(virtualizable, reader): virtualizable = cast_gcref_to_vtype(virtualizable) # Load values from the reader (see resume.py) described by # the list of numbers 'nums', and write them in their proper # place in the 'virtualizable'. for FIELDTYPE, fieldname in unroll_static_fields: - item, index = numb_next_item(numb, index) - x = reader.load_value_of_type(FIELDTYPE, item) + x = reader.load_next_value_of_type(FIELDTYPE) setattr(virtualizable, fieldname, x) for ARRAYITEMTYPE, fieldname in unroll_array_fields: lst = getattr(virtualizable, fieldname) for j in range(getlength(lst)): - item, index = numb_next_item(numb, index) - x = reader.load_value_of_type(ARRAYITEMTYPE, item) + x = reader.load_next_value_of_type(ARRAYITEMTYPE) setarrayitem(lst, j, x) - return index - def load_list_of_boxes(virtualizable, reader, vable_box, numb, index): + def load_list_of_boxes(virtualizable, reader, vable_box): virtualizable = cast_gcref_to_vtype(virtualizable) # Uses 'virtualizable' only to know the length of the arrays; # does not write anything into it. The returned list is in @@ -152,17 +148,15 @@ # the virtualizable itself. boxes = [] for FIELDTYPE, fieldname in unroll_static_fields: - item, index = numb_next_item(numb, index) - box = reader.decode_box_of_type(FIELDTYPE, item) + box = reader.next_box_of_type(FIELDTYPE) boxes.append(box) for ARRAYITEMTYPE, fieldname in unroll_array_fields: lst = getattr(virtualizable, fieldname) for j in range(getlength(lst)): - item, index = numb_next_item(numb, index) - box = reader.decode_box_of_type(ARRAYITEMTYPE, item) + box = reader.next_box_of_type(ARRAYITEMTYPE) boxes.append(box) boxes.append(vable_box) - return boxes, index + return boxes def check_boxes(virtualizable, boxes): virtualizable = cast_gcref_to_vtype(virtualizable) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit