Author: fijal
Branch: compress-numbering
Changeset: r80926:2a006fba78d9
Date: 2015-11-25 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/2a006fba78d9/
Log: write the snapshots in a better order. complicates code, but will
probably be beneficial
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "1.3.0"
-__version_info__ = (1, 3, 0)
+__version__ = "1.3.1"
+__version_info__ = (1, 3, 1)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -202,19 +202,31 @@
TAGBOX = 2
TAGVIRTUAL = 3
-UNASSIGNED = tag(-1 << 13, TAGBOX)
-UNASSIGNEDVIRTUAL = tag(-1 << 13, TAGVIRTUAL)
-NULLREF = tag(0, TAGCONST)
-UNINITIALIZED = tag(1, TAGCONST) # used for uninitialized string characters
-TAG_CONST_OFFSET = 2
+UNASSIGNED = tag(-1, TAGBOX)
+UNASSIGNEDVIRTUAL = tag(-1, TAGVIRTUAL)
+NULLREF = tag(-1, TAGCONST)
+UNINITIALIZED = tag(2, TAGCONST) # used for uninitialized string characters
+TAG_CONST_OFFSET = 0
class NumberingState(object):
- def __init__(self):
+ def __init__(self, snapshot_list):
self.liveboxes = {}
- self.current = []
+ self.current = [None] * self.count_boxes(snapshot_list)
+ self.position = len(self.current)
self.n = 0
self.v = 0
+ def count_boxes(self, lst):
+ c = 0
+ for snapshot in lst:
+ c += len(snapshot.boxes)
+ c += 2 * (len(lst) - 1)
+ return c
+
+ def append(self, item):
+ self.current[self.position] = item
+ self.position += 1
+
class ResumeDataLoopMemo(object):
def __init__(self, metainterp_sd):
@@ -273,6 +285,7 @@
v = state.v
liveboxes = state.liveboxes
length = len(boxes)
+ state.position -= length
for i in range(length):
box = boxes[i]
box = optimizer.get_box_replacement(box)
@@ -296,9 +309,10 @@
tagged = tag(n, TAGBOX)
n += 1
liveboxes[box] = tagged
- state.current.append(tagged)
+ state.append(tagged)
state.n = n
state.v = v
+ state.position -= length
# def _get_prev_snapshot(self, snapshot):
# cur_snapshot = snapshot
@@ -313,26 +327,28 @@
def number(self, optimizer, snapshot, frameinfo):
# flatten the list
- cur = snapshot
- snapshot_list = []
+ vref_snapshot = snapshot
+ cur = snapshot.prev
+ snapshot_list = [vref_snapshot]
framestack_list = []
while cur:
- if cur is not snapshot:
- framestack_list.append(frameinfo)
- frameinfo = frameinfo.prev
+ framestack_list.append(frameinfo)
+ frameinfo = frameinfo.prev
snapshot_list.append(cur)
cur = cur.prev
- snapshot_list.reverse()
- framestack_list.reverse()
- state = NumberingState()
+ state = NumberingState(snapshot_list)
- for i in range(len(snapshot_list)):
+ # we want to number snapshots starting from the back, but ending
+ # with a forward list
+ for i in range(len(snapshot_list) - 1, -1, -1):
self._number_boxes(snapshot_list[i].boxes, optimizer, state)
if i != 0:
frameinfo = framestack_list[i - 1]
jitcode_pos, pc = unpack_uint(frameinfo.packed_jitcode_pc)
- state.current.append(rffi.cast(rffi.USHORT, jitcode_pos))
- state.current.append(rffi.cast(rffi.USHORT, pc))
+ state.position -= 2
+ state.append(rffi.cast(rffi.USHORT, jitcode_pos))
+ state.append(rffi.cast(rffi.USHORT, pc))
+ state.position -= 2
numb = resumecode.create_numbering(state.current,
lltype.nullptr(resumecode.NUMBERING), 0)
@@ -1090,6 +1106,7 @@
def rebuild_from_resumedata(metainterp, storage, deadframe,
virtualizable_info, greenfield_info):
+ xxx
resumereader = ResumeDataBoxReader(storage, deadframe, metainterp)
boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info,
greenfield_info)
@@ -1480,6 +1497,7 @@
load_value_of_type._annspecialcase_ = 'specialize:arg(1)'
def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo):
+ xxx
numb = self.cur_numb
self.cur_numb = numb.prev
if self.resume_after_guard_not_forced != 2:
diff --git a/rpython/jit/metainterp/resumecode.py
b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -15,49 +15,78 @@
NUMBERING = lltype.GcStruct('Numbering',
('prev', NUMBERINGP),
('prev_index', rffi.USHORT),
- ('code', lltype.Array(rffi.UCHAR)))
+ ('code', lltype.Array(rffi.SHORT)))
NUMBERINGP.TO.become(NUMBERING)
NULL_NUMBER = lltype.nullptr(NUMBERING)
def create_numbering(lst, prev, prev_index):
- count = 0
- for item in lst:
- if item > 127:
- count += 1
- count += 1
- numb = lltype.malloc(NUMBERING, count)
- numb.prev = prev
- numb.prev_index = rffi.cast(rffi.USHORT, prev_index)
- index = 0
- for item in lst:
- assert item >= 0
- if item <= 128:
- numb.code[index] = rffi.cast(rffi.UCHAR, item)
- index += 1
- else:
- assert (item >> 8) <= 127
- numb.code[index] = rffi.cast(rffi.UCHAR, (item >> 8) |
0x80)
- numb.code[index + 1] = rffi.cast(rffi.UCHAR, item &
0xff)
- index += 2
- return numb
+ numb = lltype.malloc(NUMBERING, len(lst))
+ for i in range(len(lst)):
+ numb.code[i] = rffi.cast(rffi.SHORT, lst[i])
+ numb.prev = prev
+ numb.prev_index = rffi.cast(rffi.USHORT, prev_index)
+ return numb
+
+def _create_numbering(lst, prev, prev_index):
+ count = 0
+ for item in lst:
+ if item < 0:
+ if item < -63:
+ count += 1
+ if item > 127:
+ count += 1
+ count += 1
+ numb = lltype.malloc(NUMBERING, count)
+ numb.prev = prev
+ numb.prev_index = rffi.cast(rffi.USHORT, prev_index)
+ index = 0
+ for item in lst:
+ if 0 <= item <= 128:
+ numb.code[index] = rffi.cast(rffi.UCHAR, item)
+ index += 1
+ else:
+ assert (item >> 8) <= 63
+ if item < 0:
+ item = -item
+ if item <= 63:
+ numb.code[index] = rffi.cast(rffi.UCHAR, item | 0x40)
+ index += 1
+ else:
+ numb.code[index] = rffi.cast(rffi.UCHAR, (item >> 8) |
0x80 | 0x40)
+ numb.code[index + 1] = rffi.cast(rffi.UCHAR, item & 0xff)
+ index += 2
+ else:
+ numb.code[index] = rffi.cast(rffi.UCHAR, (item >> 8) | 0x80)
+ numb.code[index + 1] = rffi.cast(rffi.UCHAR, item & 0xff)
+ index += 2
+ return numb
def copy_from_list_to_numb(lst, numb, index):
- i = 0
- while i < len(lst):
- numb.code[i + index] = lst[i]
- i += 1
+ i = 0
+ while i < len(lst):
+ numb.code[i + index] = lst[i]
+ i += 1
def numb_next_item(numb, index):
- one = rffi.cast(lltype.Signed, numb.code[index])
- if one & 0x80:
- two = rffi.cast(lltype.Signed, numb.code[index + 1])
- return ((one & 0x7f) << 8) | two, index + 2
- return one, index + 1
+ return rffi.cast(lltype.Signed, numb.code[index]), index + 1
+
+def _numb_next_item(numb, index):
+ one = rffi.cast(lltype.Signed, numb.code[index])
+ if one & 0x40:
+ if one & 0x80:
+ two = rffi.cast(lltype.Signed, numb.code[index + 1])
+ return -(((one & ~(0x80 | 0x40)) << 8) | two), index + 2
+ else:
+ return -(one & (~0x40)), index + 1
+ if one & 0x80:
+ two = rffi.cast(lltype.Signed, numb.code[index + 1])
+ return ((one & 0x7f) << 8) | two, index + 2
+ return one, index + 1
def unpack_numbering(numb):
- l = []
- i = 0
- while i < len(numb.code):
- next, i = numb_next_item(numb, i)
- l.append(next)
- return l
+ l = []
+ i = 0
+ while i < len(numb.code):
+ next, i = numb_next_item(numb, i)
+ l.append(next)
+ return l
diff --git a/rpython/jit/metainterp/test/test_resume.py
b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -336,7 +336,7 @@
def test_simple_read_tagged_ints():
storage = Storage()
storage.rd_consts = []
- numb = Numbering(None, [tag(100, TAGINT)])
+ numb = Numbering([tag(100, TAGINT)])
storage.rd_numb = numb
#
cpu = MyCPU([])
@@ -552,17 +552,6 @@
assert jitcode_pos == 42
assert pc == 3
-def test_Numbering_create():
- l = [rffi.r_short(1), rffi.r_short(2)]
- numb = Numbering(None, l)
- assert not numb.prev
- assert list(numb.nums) == l
-
- l1 = [rffi.r_short(3)]
- numb1 = Numbering(numb, l1)
- assert numb1.prev == numb
- assert list(numb1.nums) == l1
-
def test_capture_resumedata():
b1, b2, b3 = [InputArgInt(), InputArgRef(), InputArgInt()]
c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]
@@ -869,7 +858,7 @@
tagged = memo.getconst(const)
index, tagbits = untag(tagged)
assert tagbits == TAGCONST
- assert memo.consts[index] is const
+ assert memo.consts[index - TAG_CONST_OFFSET] is const
tagged = memo.getconst(ConstInt(50000))
index2, tagbits = untag(tagged)
assert tagbits == TAGCONST
@@ -887,7 +876,7 @@
tagged = memo.getconst(const)
index, tagbits = untag(tagged)
assert tagbits == TAGCONST
- assert memo.consts[index] is const
+ assert memo.consts[index - TAG_CONST_OFFSET] is const
tagged = memo.getconst(cpu.ts.ConstRef(demo55o))
index2, tagbits = untag(tagged)
assert tagbits == TAGCONST
@@ -905,7 +894,7 @@
tagged = memo.getconst(const)
index, tagbits = untag(tagged)
assert tagbits == TAGCONST
- assert memo.consts[index] is const
+ assert memo.consts[index - TAG_CONST_OFFSET] is const
def test_ResumeDataLoopMemo_number():
b1, b2, b3, b4, b5 = [InputArgInt(), InputArgInt(), InputArgInt(),
@@ -929,8 +918,8 @@
b3: tag(2, TAGBOX)}
base = [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), tag(0, TAGBOX),
tag(2, TAGINT)]
- assert unpack_numbering(numb) == base + [
- tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(1, TAGINT), 0, 0]
+ assert unpack_numbering(numb) == [
+ tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(1, TAGINT), 0,
0] + base
numb2, liveboxes2, v = memo.number(FakeOptimizer(), snap2, frameinfo)
assert v == 0
@@ -938,8 +927,8 @@
assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b3: tag(2, TAGBOX)}
assert liveboxes2 is not liveboxes
- assert unpack_numbering(numb2) == base + [
- tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(3, TAGINT), 0, 0]
+ assert unpack_numbering(numb2) == [
+ tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(3, TAGINT), 0, 0]
+ base
assert numb2.prev == numb.prev
env3 = [c3, b3, b1, c3]
@@ -958,8 +947,8 @@
assert v == 0
assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)}
- assert unpack_numbering(numb3) == base + [tag(3, TAGINT), tag(4, TAGINT),
tag(0, TAGBOX),
- tag(3, TAGINT), 0, 0]
+ assert unpack_numbering(numb3) == [tag(3, TAGINT), tag(4, TAGINT), tag(0,
TAGBOX),
+ tag(3, TAGINT), 0, 0] + base
# virtual
env4 = [c3, b4, b1, c3]
@@ -971,8 +960,8 @@
assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b4: tag(0, TAGVIRTUAL)}
- assert unpack_numbering(numb4) == base + [tag(3, TAGINT), tag(0,
TAGVIRTUAL),
- tag(0, TAGBOX), tag(3, TAGINT), 0, 0]
+ assert unpack_numbering(numb4) == [tag(3, TAGINT), tag(0, TAGVIRTUAL),
+ tag(0, TAGBOX), tag(3, TAGINT), 0, 0] + base
env5 = [b1, b4, b5]
snap5 = Snapshot(snap4, env5)
@@ -985,8 +974,8 @@
assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)}
- assert unpack_numbering(numb5) == unpack_numbering(numb4) + [tag(0,
TAGBOX), tag(0, TAGVIRTUAL),
- tag(1, TAGVIRTUAL), 2, 1]
+ assert unpack_numbering(numb5) == [tag(0, TAGBOX), tag(0, TAGVIRTUAL),
+ tag(1, TAGVIRTUAL), 2, 1] + unpack_numbering(numb4)
def test_ResumeDataLoopMemo_number_boxes():
memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
@@ -1087,9 +1076,10 @@
assert storage.rd_snapshot is None
cpu = MyCPU([])
reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage, "deadframe")
+ _next_section(reader, sys.maxint, 1, sys.maxint, 2**16)
+ _next_section(reader, 2, 3)
+ reader.cur_index += 2 # framestack
_next_section(reader, sys.maxint, 2**16, -65)
- _next_section(reader, 2, 3)
- _next_section(reader, sys.maxint, 1, sys.maxint, 2**16)
def test_virtual_adder_memo_const_sharing():
@@ -1125,8 +1115,8 @@
return True
class MyInfo:
@staticmethod
- def enumerate_vars(callback_i, callback_r, callback_f, _):
- for index, tagged in enumerate(self.cur_numb.nums):
+ def enumerate_vars(callback_i, callback_r, callback_f, _, index):
+ for tagged in self.cur_numb.nums:
_, tag = untag(tagged)
if tag == TAGVIRTUAL:
kind = REF
@@ -1134,11 +1124,11 @@
kind = Whatever()
box = self.decode_box(tagged, kind)
if box.type == INT:
- callback_i(index, index)
+ index = callback_i(index, index)
elif box.type == REF:
- callback_r(index, index)
+ index = callback_r(index, index)
elif box.type == FLOAT:
- callback_f(index, index)
+ index = callback_f(index, index)
else:
assert 0
self._prepare_next_section(MyInfo())
diff --git a/rpython/jit/metainterp/test/test_resumecode.py
b/rpython/jit/metainterp/test/test_resumecode.py
--- a/rpython/jit/metainterp/test/test_resumecode.py
+++ b/rpython/jit/metainterp/test/test_resumecode.py
@@ -1,18 +1,16 @@
-from rpython.jit.metainterp.resumecode import NUMBERING
+from rpython.jit.metainterp.resumecode import NUMBERING, NULL_NUMBER
from rpython.jit.metainterp.resumecode import create_numbering,\
- unpack_numbering, copy_from_list_to_numb
+ unpack_numbering, copy_from_list_to_numb
from rpython.rtyper.lltypesystem import lltype
def test_pack_unpack():
- examples = [
- [1, 2, 3, 4, 257, 10000, 13, 15],
- [1, 2, 3, 4],
- range(1, 10, 2),
- [13000, 12000, 10000, 256, 255, 254, 257]
- ]
- for l in examples:
- lst = create_numbering(l)
- n = lltype.malloc(NUMBERING, len(lst))
- copy_from_list_to_numb(lst, n, 0)
- assert unpack_numbering(n) == l
+ examples = [
+ [1, 2, 3, 4, 257, 10000, 13, 15],
+ [1, 2, 3, 4],
+ range(1, 10, 2),
+ [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
+ ]
+ for l in examples:
+ n = create_numbering(l, NULL_NUMBER, 0)
+ assert unpack_numbering(n) == l
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
