Author: Ronan Lamy <ronan.l...@gmail.com>
Branch: cpyext-ext
Changeset: r82438:7fad651c7daf
Date: 2016-02-23 14:39 +0100
http://bitbucket.org/pypy/pypy/changeset/7fad651c7daf/
Log: hg merge default
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -128,6 +128,7 @@
Fix SSL tests by importing cpython's patch
+
.. branch: remove-getfield-pure
Remove pure variants of ``getfield_gc_*`` operations from the JIT. Relevant
@@ -163,3 +164,10 @@
.. branch: windows-vmprof-support
vmprof should work on Windows.
+
+
+.. branch: reorder-map-attributes
+
+When creating instances and adding attributes in several different orders
+depending on some condition, the JIT would create too much code. This is now
+fixed.
\ No newline at end of file
diff --git a/pypy/module/imp/test/test_import.py
b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1061,12 +1061,12 @@
py.test.skip("unresolved issues with win32 shell quoting rules")
from pypy.interpreter.test.test_zpy import pypypath
extrapath = udir.ensure("pythonpath", dir=1)
- extrapath.join("urllib.py").write("print 42\n")
+ extrapath.join("sched.py").write("print 42\n")
old = os.environ.get('PYTHONPATH', None)
oldlang = os.environ.pop('LANG', None)
try:
os.environ['PYTHONPATH'] = str(extrapath)
- output = py.process.cmdexec('''"%s" "%s" -c "import urllib"''' %
+ output = py.process.cmdexec('''"%s" "%s" -c "import sched"''' %
(sys.executable, pypypath))
assert output.strip() == '42'
finally:
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py
b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -28,7 +28,6 @@
guard_true(i14, descr=...)
guard_not_invalidated(descr=...)
i16 = int_eq(i6, %d)
- guard_false(i16, descr=...)
i15 = int_mod(i6, i10)
i17 = int_rshift(i15, %d)
i18 = int_and(i10, i17)
@@ -68,7 +67,6 @@
guard_true(i11, descr=...)
guard_not_invalidated(descr=...)
i13 = int_eq(i6, %d) # value provided below
- guard_false(i13, descr=...)
i15 = int_mod(i6, 10)
i17 = int_rshift(i15, %d) # value provided below
i18 = int_and(10, i17)
@@ -144,43 +142,6 @@
jump(..., descr=...)
""")
- def test_getattr_promote(self):
- def main(n):
- class A(object):
- def meth_a(self):
- return 1
- def meth_b(self):
- return 2
- a = A()
-
- l = ['a', 'b']
- s = 0
- for i in range(n):
- name = 'meth_' + l[i & 1]
- meth = getattr(a, name) # ID: getattr
- s += meth()
- return s
-
- log = self.run(main, [1000])
- assert log.result == main(1000)
- loops = log.loops_by_filename(self.filepath)
- assert len(loops) == 1
- for loop in loops:
- assert loop.match_by_id('getattr','''
- guard_not_invalidated?
- i32 = strlen(p31)
- i34 = int_add(5, i32)
- p35 = newstr(i34)
- strsetitem(p35, 0, 109)
- strsetitem(p35, 1, 101)
- strsetitem(p35, 2, 116)
- strsetitem(p35, 3, 104)
- strsetitem(p35, 4, 95)
- copystrcontent(p31, p35, 0, 5, i32)
- i49 =
call_i(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35,
ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>)
- guard_value(i49, 1, descr=...)
- ''')
-
def test_remove_duplicate_method_calls(self):
def main(n):
lst = []
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -1,4 +1,4 @@
-import weakref
+import weakref, sys
from rpython.rlib import jit, objectmodel, debug, rerased
from rpython.rlib.rarithmetic import intmask, r_uint
@@ -12,6 +12,11 @@
from pypy.objspace.std.typeobject import MutableCell
+erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage item")
+erase_map, unerase_map = rerased.new_erasing_pair("map")
+erase_list, unerase_list = rerased.new_erasing_pair("mapdict storage list")
+
+
# ____________________________________________________________
# attribute shapes
@@ -20,6 +25,7 @@
# note: we use "x * NUM_DIGITS_POW2" instead of "x << NUM_DIGITS" because
# we want to propagate knowledge that the result cannot be negative
+
class AbstractAttribute(object):
_immutable_fields_ = ['terminator']
cache_attrs = None
@@ -151,29 +157,124 @@
cache[name, index] = attr
return attr
+ @jit.elidable
+ def _get_cache_attr(self, name, index):
+ key = name, index
+ # this method is not actually elidable, but it's fine anyway
+ if self.cache_attrs is not None:
+ return self.cache_attrs.get(key, None)
+ return None
+
+ def add_attr(self, obj, name, index, w_value):
+ self._reorder_and_add(obj, name, index, w_value)
+ if not jit.we_are_jitted():
+ oldattr = self
+ attr = obj._get_mapdict_map()
+ size_est = (oldattr._size_estimate + attr.size_estimate()
+ - oldattr.size_estimate())
+ assert size_est >= (oldattr.length() * NUM_DIGITS_POW2)
+ oldattr._size_estimate = size_est
+
+ def _add_attr_without_reordering(self, obj, name, index, w_value):
+ attr = self._get_new_attr(name, index)
+ attr._switch_map_and_write_storage(obj, w_value)
+
+ @jit.unroll_safe
+ def _switch_map_and_write_storage(self, obj, w_value):
+ if self.length() > obj._mapdict_storage_length():
+ # note that self.size_estimate() is always at least self.length()
+ new_storage = [None] * self.size_estimate()
+ for i in range(obj._mapdict_storage_length()):
+ new_storage[i] = obj._mapdict_read_storage(i)
+ obj._set_mapdict_storage_and_map(new_storage, self)
+
+ # the order is important here: first change the map, then the storage,
+ # for the benefit of the special subclasses
+ obj._set_mapdict_map(self)
+ obj._mapdict_write_storage(self.storageindex, w_value)
+
+
+ @jit.elidable
+ def _find_branch_to_move_into(self, name, index):
+ # walk up the map chain to find an ancestor with lower order that
+ # already has the current name as a child inserted
+ current_order = sys.maxint
+ number_to_readd = 0
+ current = self
+ key = (name, index)
+ while True:
+ attr = None
+ if current.cache_attrs is not None:
+ attr = current.cache_attrs.get(key, None)
+ if attr is None or attr.order > current_order:
+ # we reached the top, so we didn't find it anywhere,
+ # just add it to the top attribute
+ if not isinstance(current, PlainAttribute):
+ return 0, self._get_new_attr(name, index)
+
+ else:
+ return number_to_readd, attr
+ # if not found try parent
+ number_to_readd += 1
+ current_order = current.order
+ current = current.back
+
@jit.look_inside_iff(lambda self, obj, name, index, w_value:
jit.isconstant(self) and
jit.isconstant(name) and
jit.isconstant(index))
- def add_attr(self, obj, name, index, w_value):
- attr = self._get_new_attr(name, index)
- oldattr = obj._get_mapdict_map()
- if not jit.we_are_jitted():
- size_est = (oldattr._size_estimate + attr.size_estimate()
- - oldattr.size_estimate())
- assert size_est >= (oldattr.length() * NUM_DIGITS_POW2)
- oldattr._size_estimate = size_est
- if attr.length() > obj._mapdict_storage_length():
- # note that attr.size_estimate() is always at least attr.length()
- new_storage = [None] * attr.size_estimate()
- for i in range(obj._mapdict_storage_length()):
- new_storage[i] = obj._mapdict_read_storage(i)
- obj._set_mapdict_storage_and_map(new_storage, attr)
+ def _reorder_and_add(self, obj, name, index, w_value):
+ # the idea is as follows: the subtrees of any map are ordered by
+ # insertion. the invariant is that subtrees that are inserted later
+ # must not contain the name of the attribute of any earlier inserted
+ # attribute anywhere
+ # m______
+ # inserted first / \ ... \ further attributes
+ # attrname a 0/ 1\ n\
+ # m a must not appear here anywhere
+ #
+ # when inserting a new attribute in an object we check whether any
+ # parent of lower order has seen that attribute yet. if yes, we follow
+ # that branch. if not, we normally append that attribute. When we
+ # follow a prior branch, we necessarily remove some attributes to be
+ # able to do that. They need to be re-added, which has to follow the
+ # reordering procedure recusively.
- # the order is important here: first change the map, then the storage,
- # for the benefit of the special subclasses
- obj._set_mapdict_map(attr)
- obj._mapdict_write_storage(attr.storageindex, w_value)
+ # we store the to-be-readded attribute in the stack, with the map and
+ # the value paired up those are lazily initialized to a list large
+ # enough to store all current attributes
+ stack = None
+ stack_index = 0
+ while True:
+ current = self
+ number_to_readd = 0
+ number_to_readd, attr = self._find_branch_to_move_into(name, index)
+ # we found the attributes further up, need to save the
+ # previous values of the attributes we passed
+ if number_to_readd:
+ if stack is None:
+ stack = [erase_map(None)] * (self.length() * 2)
+ current = self
+ for i in range(number_to_readd):
+ assert isinstance(current, PlainAttribute)
+ w_self_value = obj._mapdict_read_storage(
+ current.storageindex)
+ stack[stack_index] = erase_map(current)
+ stack[stack_index + 1] = erase_item(w_self_value)
+ stack_index += 2
+ current = current.back
+ attr._switch_map_and_write_storage(obj, w_value)
+
+ if not stack_index:
+ return
+
+ # readd the current top of the stack
+ stack_index -= 2
+ next_map = unerase_map(stack[stack_index])
+ w_value = unerase_item(stack[stack_index + 1])
+ name = next_map.name
+ index = next_map.index
+ self = obj._get_mapdict_map()
def materialize_r_dict(self, space, obj, dict_w):
raise NotImplementedError("abstract base class")
@@ -279,7 +380,7 @@
return Terminator.set_terminator(self, obj, terminator)
class PlainAttribute(AbstractAttribute):
- _immutable_fields_ = ['name', 'index', 'storageindex', 'back',
'ever_mutated?']
+ _immutable_fields_ = ['name', 'index', 'storageindex', 'back',
'ever_mutated?', 'order']
def __init__(self, name, index, back):
AbstractAttribute.__init__(self, back.space, back.terminator)
@@ -289,6 +390,7 @@
self.back = back
self._size_estimate = self.length() * NUM_DIGITS_POW2
self.ever_mutated = False
+ self.order = len(back.cache_attrs) if back.cache_attrs else 0
def _copy_attr(self, obj, new_obj):
w_value = self.read(obj, self.name, self.index)
@@ -542,9 +644,6 @@
memo_get_subclass_of_correct_size._annspecialcase_ = "specialize:memo"
_subclass_cache = {}
-erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage item")
-erase_list, unerase_list = rerased.new_erasing_pair("mapdict storage list")
-
def _make_subclass_size_n(supercls, n):
from rpython.rlib import unroll
rangen = unroll.unrolling_iterable(range(n))
diff --git a/pypy/objspace/std/test/test_mapdict.py
b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -107,6 +107,153 @@
assert obj2.getdictvalue(space, "b") == 60
assert obj2.map is obj.map
+def test_insert_different_orders():
+ cls = Class()
+ obj = cls.instantiate()
+ obj.setdictvalue(space, "a", 10)
+ obj.setdictvalue(space, "b", 20)
+
+ obj2 = cls.instantiate()
+ obj2.setdictvalue(space, "b", 30)
+ obj2.setdictvalue(space, "a", 40)
+
+ assert obj.map is obj2.map
+
+def test_insert_different_orders_2():
+ cls = Class()
+ obj = cls.instantiate()
+ obj2 = cls.instantiate()
+
+ obj.setdictvalue(space, "a", 10)
+
+ obj2.setdictvalue(space, "b", 20)
+ obj2.setdictvalue(space, "a", 30)
+
+ obj.setdictvalue(space, "b", 40)
+ assert obj.map is obj2.map
+
+def test_insert_different_orders_3():
+ cls = Class()
+ obj = cls.instantiate()
+ obj2 = cls.instantiate()
+ obj3 = cls.instantiate()
+ obj4 = cls.instantiate()
+ obj5 = cls.instantiate()
+ obj6 = cls.instantiate()
+
+ obj.setdictvalue(space, "a", 10)
+ obj.setdictvalue(space, "b", 20)
+ obj.setdictvalue(space, "c", 30)
+
+ obj2.setdictvalue(space, "a", 30)
+ obj2.setdictvalue(space, "c", 40)
+ obj2.setdictvalue(space, "b", 50)
+
+ obj3.setdictvalue(space, "c", 30)
+ obj3.setdictvalue(space, "a", 40)
+ obj3.setdictvalue(space, "b", 50)
+
+ obj4.setdictvalue(space, "c", 30)
+ obj4.setdictvalue(space, "b", 40)
+ obj4.setdictvalue(space, "a", 50)
+
+ obj5.setdictvalue(space, "b", 30)
+ obj5.setdictvalue(space, "a", 40)
+ obj5.setdictvalue(space, "c", 50)
+
+ obj6.setdictvalue(space, "b", 30)
+ obj6.setdictvalue(space, "c", 40)
+ obj6.setdictvalue(space, "a", 50)
+
+ assert obj.map is obj2.map
+ assert obj.map is obj3.map
+ assert obj.map is obj4.map
+ assert obj.map is obj5.map
+ assert obj.map is obj6.map
+
+
+def test_insert_different_orders_4():
+ cls = Class()
+ obj = cls.instantiate()
+ obj2 = cls.instantiate()
+
+ obj.setdictvalue(space, "a", 10)
+ obj.setdictvalue(space, "b", 20)
+ obj.setdictvalue(space, "c", 30)
+ obj.setdictvalue(space, "d", 40)
+
+ obj2.setdictvalue(space, "d", 50)
+ obj2.setdictvalue(space, "c", 50)
+ obj2.setdictvalue(space, "b", 50)
+ obj2.setdictvalue(space, "a", 50)
+
+ assert obj.map is obj2.map
+
+def test_insert_different_orders_5():
+ cls = Class()
+ obj = cls.instantiate()
+ obj2 = cls.instantiate()
+
+ obj.setdictvalue(space, "a", 10)
+ obj.setdictvalue(space, "b", 20)
+ obj.setdictvalue(space, "c", 30)
+ obj.setdictvalue(space, "d", 40)
+
+ obj2.setdictvalue(space, "d", 50)
+ obj2.setdictvalue(space, "c", 50)
+ obj2.setdictvalue(space, "b", 50)
+ obj2.setdictvalue(space, "a", 50)
+
+ obj3 = cls.instantiate()
+ obj3.setdictvalue(space, "d", 50)
+ obj3.setdictvalue(space, "c", 50)
+ obj3.setdictvalue(space, "b", 50)
+ obj3.setdictvalue(space, "a", 50)
+
+ assert obj.map is obj3.map
+
+
+def test_bug_stack_overflow_insert_attributes():
+ cls = Class()
+ obj = cls.instantiate()
+
+ for i in range(1000):
+ obj.setdictvalue(space, str(i), i)
+
+
+def test_insert_different_orders_perm():
+ from itertools import permutations
+ cls = Class()
+ seen_maps = {}
+ for preexisting in ['', 'x', 'xy']:
+ for i, attributes in enumerate(permutations("abcdef")):
+ obj = cls.instantiate()
+ for i, attr in enumerate(preexisting):
+ obj.setdictvalue(space, attr, i*1000)
+ key = preexisting
+ for j, attr in enumerate(attributes):
+ obj.setdictvalue(space, attr, i*10+j)
+ key = "".join(sorted(key+attr))
+ if key in seen_maps:
+ assert obj.map is seen_maps[key]
+ else:
+ seen_maps[key] = obj.map
+
+ print len(seen_maps)
+
+
+def test_bug_infinite_loop():
+ cls = Class()
+ obj = cls.instantiate()
+ obj.setdictvalue(space, "e", 1)
+ obj2 = cls.instantiate()
+ obj2.setdictvalue(space, "f", 2)
+ obj3 = cls.instantiate()
+ obj3.setdictvalue(space, "a", 3)
+ obj3.setdictvalue(space, "e", 4)
+ obj3.setdictvalue(space, "f", 5)
+
+
def test_attr_immutability(monkeypatch):
cls = Class()
obj = cls.instantiate()
@@ -359,9 +506,15 @@
class TestMapDictImplementation(BaseTestRDictImplementation):
StrategyClass = MapDictStrategy
get_impl = get_impl
+ def test_setdefault_fast(self):
+ # mapdict can't pass this, which is fine
+ pass
class TestDevolvedMapDictImplementation(BaseTestDevolvedDictImplementation):
get_impl = get_impl
StrategyClass = MapDictStrategy
+ def test_setdefault_fast(self):
+ # mapdict can't pass this, which is fine
+ pass
# ___________________________________________________________
# tests that check the obj interface after the dict has devolved
@@ -1132,3 +1285,7 @@
class TestMapDictImplementationUsingnewdict(BaseTestRDictImplementation):
StrategyClass = MapDictStrategy
# NB: the get_impl method is not overwritten here, as opposed to above
+
+ def test_setdefault_fast(self):
+ # mapdict can't pass this, which is fine
+ pass
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+# hypothesis is used for test generation on untranslated jit tests
+hypothesis
+
diff --git a/rpython/jit/backend/ppc/test/test_runner.py
b/rpython/jit/backend/ppc/test/test_runner.py
--- a/rpython/jit/backend/ppc/test/test_runner.py
+++ b/rpython/jit/backend/ppc/test/test_runner.py
@@ -134,7 +134,7 @@
def test_debugger_on(self):
py.test.skip("XXX")
- from pypy.rlib import debug
+ from rpython.rlib import debug
targettoken, preambletoken = TargetToken(), TargetToken()
loop = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -28,7 +28,7 @@
# setup rd data
fi0 = resume.FrameInfo(None, FakeJitCode(), 11)
snapshot0 = resume.Snapshot(None, [b0])
- op.rd_snapshot = resume.Snapshot(snapshot0, [b1])
+ op.rd_snapshot = resume.TopSnapshot(snapshot0, [], [b1])
op.rd_frame_info_list = resume.FrameInfo(fi0, FakeJitCode(), 33)
#
opt.store_final_boxes_in_guard(op, [])
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py
b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -506,14 +506,15 @@
index = 0
if op.is_guard():
- op.rd_snapshot = resume.Snapshot(None, op.getfailargs())
+ op.rd_snapshot = resume.TopSnapshot(
+ resume.Snapshot(None, op.getfailargs()), [], [])
op.rd_frame_info_list = resume.FrameInfo(None, FakeJitCode(), 11)
def add_guard_future_condition(self, res):
# invent a GUARD_FUTURE_CONDITION to not have to change all tests
if res.operations[-1].getopnum() == rop.JUMP:
guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None)
- guard.rd_snapshot = resume.Snapshot(None, [])
+ guard.rd_snapshot = resume.TopSnapshot(None, [], [])
res.operations.insert(-1, guard)
def assert_equal(self, optimized, expected, text_right=None):
diff --git a/rpython/jit/metainterp/optimizeopt/util.py
b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -8,7 +8,7 @@
from rpython.jit.metainterp import resoperation
from rpython.rlib.debug import make_sure_not_resized
from rpython.jit.metainterp.resoperation import rop
-from rpython.jit.metainterp.resume import Snapshot, AccumInfo
+from rpython.jit.metainterp.resume import AccumInfo
# ____________________________________________________________
# Misc. utilities
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -27,6 +27,13 @@
self.prev = prev
self.boxes = boxes
+class TopSnapshot(Snapshot):
+ __slots__ = ('vable_boxes',)
+
+ def __init__(self, prev, boxes, vable_boxes):
+ Snapshot.__init__(self, prev, boxes)
+ self.vable_boxes = vable_boxes
+
def combine_uint(index1, index2):
assert 0 <= index1 < 65536
assert 0 <= index2 < 65536
@@ -127,9 +134,11 @@
snapshot_storage):
n = len(framestack) - 1
if virtualizable_boxes is not None:
- boxes = virtualref_boxes + virtualizable_boxes
+ virtualizable_boxes = ([virtualizable_boxes[-1]] +
+ virtualizable_boxes[:-1])
else:
- boxes = virtualref_boxes[:]
+ virtualizable_boxes = []
+ virtualref_boxes = virtualref_boxes[:]
if n >= 0:
top = framestack[n]
_ensure_parent_resumedata(framestack, n)
@@ -138,11 +147,12 @@
snapshot_storage.rd_frame_info_list = frame_info_list
snapshot = Snapshot(top.parent_resumedata_snapshot,
top.get_list_of_active_boxes(False))
- snapshot = Snapshot(snapshot, boxes)
+ snapshot = TopSnapshot(snapshot, virtualref_boxes, virtualizable_boxes)
snapshot_storage.rd_snapshot = snapshot
else:
snapshot_storage.rd_frame_info_list = None
- snapshot_storage.rd_snapshot = Snapshot(None, boxes)
+ snapshot_storage.rd_snapshot = TopSnapshot(None, virtualref_boxes,
+ virtualizable_boxes)
PENDINGFIELDSTRUCT = lltype.Struct('PendingField',
('lldescr', OBJECTPTR),
@@ -200,10 +210,12 @@
self.v = 0
def count_boxes(self, lst):
- c = 0
+ snapshot = lst[0]
+ assert isinstance(snapshot, TopSnapshot)
+ c = len(snapshot.vable_boxes)
for snapshot in lst:
c += len(snapshot.boxes)
- c += 2 * (len(lst) - 1)
+ c += 2 * (len(lst) - 1) + 1 + 1
return c
def append(self, item):
@@ -294,13 +306,11 @@
state.append(tagged)
state.n = n
state.v = v
- state.position -= length + 2
- def number(self, optimizer, snapshot, frameinfo):
+ def number(self, optimizer, topsnapshot, frameinfo):
# flatten the list
- vref_snapshot = snapshot
- cur = snapshot.prev
- snapshot_list = [vref_snapshot]
+ cur = topsnapshot.prev
+ snapshot_list = [topsnapshot]
framestack_list = []
while cur:
framestack_list.append(frameinfo)
@@ -311,19 +321,30 @@
# we want to number snapshots starting from the back, but ending
# with a forward list
- for i in range(len(snapshot_list) - 1, -1, -1):
- state.position -= len(snapshot_list[i].boxes)
- if i != 0:
- frameinfo = framestack_list[i - 1]
- jitcode_pos, pc = unpack_uint(frameinfo.packed_jitcode_pc)
- state.position -= 2
- state.append(rffi.cast(rffi.SHORT, jitcode_pos))
- state.append(rffi.cast(rffi.SHORT, pc))
+ for i in range(len(snapshot_list) - 1, 0, -1):
+ state.position -= len(snapshot_list[i].boxes) + 2
+ frameinfo = framestack_list[i - 1]
+ jitcode_pos, pc = unpack_uint(frameinfo.packed_jitcode_pc)
+ state.append(rffi.cast(rffi.SHORT, jitcode_pos))
+ state.append(rffi.cast(rffi.SHORT, pc))
self._number_boxes(snapshot_list[i].boxes, optimizer, state)
+ state.position -= len(snapshot_list[i].boxes) + 2
- numb = resumecode.create_numbering(state.current,
- len(vref_snapshot.boxes))
+ assert isinstance(topsnapshot, TopSnapshot)
+ special_boxes_size = (1 + len(topsnapshot.vable_boxes) +
+ 1 + len(topsnapshot.boxes))
+ assert state.position == special_boxes_size
+ state.position = 0
+ state.append(rffi.cast(rffi.SHORT, len(topsnapshot.vable_boxes)))
+ self._number_boxes(topsnapshot.vable_boxes, optimizer, state)
+ n = len(topsnapshot.boxes)
+ assert not (n & 1)
+ state.append(rffi.cast(rffi.SHORT, n >> 1))
+ self._number_boxes(topsnapshot.boxes, optimizer, state)
+ assert state.position == special_boxes_size
+
+ numb = resumecode.create_numbering(state.current)
return numb, state.liveboxes, state.v
def forget_numberings(self):
@@ -1113,48 +1134,42 @@
self.boxes_f = boxes_f
self._prepare_next_section(info)
- def consume_virtualizable_boxes(self, vinfo):
+ def consume_virtualizable_boxes(self, vinfo, index):
# we have to ignore the initial part of 'nums' (containing vrefs),
# find the virtualizable from nums[-1], and use it to know how many
# boxes of which type we have to return. This does not write
# anything into the virtualizable.
numb = self.numb
- first_snapshot_size = rffi.cast(lltype.Signed,
numb.first_snapshot_size)
- item, _ = resumecode.numb_next_item(numb, first_snapshot_size - 1)
+ item, index = resumecode.numb_next_item(numb, index)
virtualizablebox = self.decode_ref(item)
- index = first_snapshot_size -
vinfo.get_total_size(virtualizablebox.getref_base()) - 1
virtualizable = vinfo.unwrap_virtualizable_box(virtualizablebox)
return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox,
numb, index)
- def consume_virtualref_boxes(self, end):
+ def consume_virtualref_boxes(self, index):
# Returns a list of boxes, assumed to be all BoxPtrs.
# We leave up to the caller to call vrefinfo.continue_tracing().
- assert (end & 1) == 0
+ size, index = resumecode.numb_next_item(self.numb, index)
+ if size == 0:
+ return [], index
lst = []
- self.cur_index = 0
- for i in range(end):
- item, self.cur_index = resumecode.numb_next_item(self.numb,
- self.cur_index)
+ for i in range(size * 2):
+ item, index = resumecode.numb_next_item(self.numb, index)
lst.append(self.decode_ref(item))
- return lst
+ return lst, index
def consume_vref_and_vable_boxes(self, vinfo, ginfo):
- first_snapshot_size = rffi.cast(lltype.Signed,
- self.numb.first_snapshot_size)
+ vable_size, index = resumecode.numb_next_item(self.numb, 0)
if vinfo is not None:
- virtualizable_boxes = self.consume_virtualizable_boxes(vinfo)
- end = first_snapshot_size - len(virtualizable_boxes)
+ virtualizable_boxes, index =
self.consume_virtualizable_boxes(vinfo,
+
index)
elif ginfo is not None:
- item, self.cur_index = resumecode.numb_next_item(self.numb,
- first_snapshot_size - 1)
+ item, index = resumecode.numb_next_item(self.numb, index)
virtualizable_boxes = [self.decode_ref(item)]
- end = first_snapshot_size - 1
else:
- end = first_snapshot_size
virtualizable_boxes = None
- virtualref_boxes = self.consume_virtualref_boxes(end)
- self.cur_index = rffi.cast(lltype.Signed,
self.numb.first_snapshot_size)
+ virtualref_boxes, index = self.consume_virtualref_boxes(index)
+ self.cur_index = index
return virtualizable_boxes, virtualref_boxes
def allocate_with_vtable(self, descr=None):
@@ -1429,39 +1444,36 @@
info = blackholeinterp.get_current_position_info()
self._prepare_next_section(info)
- def consume_virtualref_info(self, vrefinfo, end):
+ def consume_virtualref_info(self, vrefinfo, index):
# we have to decode a list of references containing pairs
- # [..., virtual, vref, ...] stopping at 'end'
- if vrefinfo is None:
- assert end == 0
- return
- assert (end & 1) == 0
- self.cur_index = 0
- for i in range(0, end, 2):
- virtual_item, self.cur_index = resumecode.numb_next_item(
- self.numb, self.cur_index)
- vref_item, self.cur_index = resumecode.numb_next_item(
- self.numb, self.cur_index)
+ # [..., virtual, vref, ...] and returns the index at the end
+ size, index = resumecode.numb_next_item(self.numb, index)
+ if vrefinfo is None or size == 0:
+ assert size == 0
+ return index
+ for i in range(size):
+ virtual_item, index = resumecode.numb_next_item(
+ self.numb, index)
+ vref_item, index = resumecode.numb_next_item(
+ self.numb, index)
virtual = self.decode_ref(virtual_item)
vref = self.decode_ref(vref_item)
# For each pair, we store the virtual inside the vref.
vrefinfo.continue_tracing(vref, virtual)
+ return index
- def consume_vable_info(self, vinfo):
+ def consume_vable_info(self, vinfo, index):
# we have to ignore the initial part of 'nums' (containing vrefs),
# find the virtualizable from nums[-1], load all other values
# from the CPU stack, and copy them into the virtualizable
numb = self.numb
- first_snapshot_size = rffi.cast(lltype.Signed,
numb.first_snapshot_size)
- item, _ = resumecode.numb_next_item(self.numb,
- first_snapshot_size - 1)
+ item, index = resumecode.numb_next_item(self.numb, index)
virtualizable = self.decode_ref(item)
- start_index = first_snapshot_size - 1 -
vinfo.get_total_size(virtualizable)
# just reset the token, we'll force it later
vinfo.reset_token_gcref(virtualizable)
- vinfo.write_from_resume_data_partial(virtualizable, self, start_index,
- numb)
- return start_index
+ index = vinfo.write_from_resume_data_partial(virtualizable, self,
+ index, numb)
+ return index
def load_value_of_type(self, TYPE, tagged):
from rpython.jit.metainterp.warmstate import specialize_value
@@ -1478,14 +1490,18 @@
load_value_of_type._annspecialcase_ = 'specialize:arg(1)'
def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo):
+ vable_size, index = resumecode.numb_next_item(self.numb, 0)
if self.resume_after_guard_not_forced != 2:
- end_vref = rffi.cast(lltype.Signed, self.numb.first_snapshot_size)
if vinfo is not None:
- end_vref = self.consume_vable_info(vinfo)
+ index = self.consume_vable_info(vinfo, index)
if ginfo is not None:
- end_vref -= 1
- self.consume_virtualref_info(vrefinfo, end_vref)
- self.cur_index = rffi.cast(lltype.Signed,
self.numb.first_snapshot_size)
+ _, index = resumecode.numb_next_item(self.numb, index)
+ index = self.consume_virtualref_info(vrefinfo, index)
+ else:
+ index = resumecode.numb_next_n_items(self.numb, vable_size, index)
+ vref_size, index = resumecode.numb_next_item(self.numb, index)
+ index = resumecode.numb_next_n_items(self.numb, vref_size * 2,
index)
+ self.cur_index = index
def allocate_with_vtable(self, descr=None):
from rpython.jit.metainterp.executor import exec_new_with_vtable
diff --git a/rpython/jit/metainterp/resumecode.py
b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -1,96 +1,74 @@
""" Resume bytecode. It goes as following:
-<numb> <numb> <pc> <jitcode> <numb> <numb> <numb> <pc> <jitcode>
+ [<length> <virtualizable object> <numb> <numb> <numb>] if vinfo is not
None
+ -OR-
+ [1 <ginfo object>] if ginfo is not
None
+ -OR-
+ [0] if both are None
-until the length of the array.
+ [<length> <virtual> <vref> <virtual> <vref>] for virtualrefs
-The interface is only create_numbering/numb_next_item, but! there is a trick
-that uses first_snapshot_size + some knowledge about inside to decode
-virtualref/virtualizable_fields/virtualizable in that order in resume.py.
+ [<pc> <jitcode> <numb> <numb> <numb>] the frames
+ [<pc> <jitcode> <numb> <numb>]
+ ...
-If the algorithm changes, the part about how to find where virtualizable
-and virtualrefs are to be found
+ until the length of the array.
"""
from rpython.rtyper.lltypesystem import rffi, lltype
NUMBERINGP = lltype.Ptr(lltype.GcForwardReference())
NUMBERING = lltype.GcStruct('Numbering',
-# ('prev', NUMBERINGP),
-# ('prev_index', rffi.USHORT),
- ('first_snapshot_size', rffi.USHORT), # ugh, ugly
- ('code', lltype.Array(rffi.SHORT)))
+ ('code', lltype.Array(rffi.UCHAR)))
NUMBERINGP.TO.become(NUMBERING)
NULL_NUMBER = lltype.nullptr(NUMBERING)
-# this is the actually used version
+def create_numbering(lst):
+ result = []
+ for item in lst:
+ item = rffi.cast(lltype.Signed, item)
+ item *= 2
+ if item < 0:
+ item = -1 - item
-def create_numbering(lst, first_snapshot_size):
- numb = lltype.malloc(NUMBERING, len(lst))
- for i in range(len(lst)):
- numb.code[i] = rffi.cast(rffi.SHORT, lst[i])
- numb.first_snapshot_size = rffi.cast(rffi.USHORT, first_snapshot_size)
+ assert item >= 0
+ if item < 2**7:
+ result.append(rffi.cast(rffi.UCHAR, item))
+ elif item < 2**14:
+ result.append(rffi.cast(rffi.UCHAR, item | 0x80))
+ result.append(rffi.cast(rffi.UCHAR, item >> 7))
+ else:
+ assert item < 2**16
+ result.append(rffi.cast(rffi.UCHAR, item | 0x80))
+ result.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80))
+ result.append(rffi.cast(rffi.UCHAR, item >> 14))
+
+ numb = lltype.malloc(NUMBERING, len(result))
+ for i in range(len(result)):
+ numb.code[i] = result[i]
return numb
def numb_next_item(numb, index):
- return rffi.cast(lltype.Signed, numb.code[index]), index + 1
+ value = rffi.cast(lltype.Signed, numb.code[index])
+ index += 1
+ if value & (2**7):
+ value &= 2**7 - 1
+ value |= rffi.cast(lltype.Signed, numb.code[index]) << 7
+ index += 1
+ if value & (2**14):
+ value &= 2**14 - 1
+ value |= rffi.cast(lltype.Signed, numb.code[index]) << 14
+ index += 1
+ if value & 1:
+ value = -1 - value
+ value >>= 1
+ return value, index
-# this is the version that can be potentially used
-
-def _create_numbering(lst, prev, prev_index, first_snapshot_size):
- count = 0
- for item in lst:
- if item < 0:
- if item < -63:
- count += 1
- if item > 127:
- count += 1
- count += 1
- numb = lltype.malloc(NUMBERING, count)
- numb.prev = prev
- numb.prev_index = rffi.cast(rffi.USHORT, prev_index)
- numb.first_snapshot_size = rffi.cast(rffi.USHORT, first_snapshot_size)
- index = 0
- for item in lst:
- if 0 <= item <= 128:
- numb.code[index] = rffi.cast(rffi.UCHAR, item)
- index += 1
- else:
- assert (item >> 8) <= 63
- if item < 0:
- item = -item
- if item <= 63:
- numb.code[index] = rffi.cast(rffi.UCHAR, item | 0x40)
- index += 1
- else:
- numb.code[index] = rffi.cast(rffi.UCHAR, (item >> 8) |
0x80 | 0x40)
- numb.code[index + 1] = rffi.cast(rffi.UCHAR, item & 0xff)
- index += 2
- else:
- numb.code[index] = rffi.cast(rffi.UCHAR, (item >> 8) | 0x80)
- numb.code[index + 1] = rffi.cast(rffi.UCHAR, item & 0xff)
- index += 2
- return numb
-
-def copy_from_list_to_numb(lst, numb, index):
- i = 0
- while i < len(lst):
- numb.code[i + index] = lst[i]
- i += 1
-
-def _numb_next_item(numb, index):
- one = rffi.cast(lltype.Signed, numb.code[index])
- if one & 0x40:
- if one & 0x80:
- two = rffi.cast(lltype.Signed, numb.code[index + 1])
- return -(((one & ~(0x80 | 0x40)) << 8) | two), index + 2
- else:
- return -(one & (~0x40)), index + 1
- if one & 0x80:
- two = rffi.cast(lltype.Signed, numb.code[index + 1])
- return ((one & 0x7f) << 8) | two, index + 2
- return one, index + 1
+def numb_next_n_items(numb, size, index):
+ for i in range(size):
+ _, index = numb_next_item(numb, index)
+ return index
def unpack_numbering(numb):
l = []
diff --git a/rpython/jit/metainterp/test/strategies.py
b/rpython/jit/metainterp/test/strategies.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/strategies.py
@@ -0,0 +1,13 @@
+
+import sys
+from hypothesis import strategies
+from rpython.jit.metainterp.resoperation import InputArgInt
+from rpython.jit.metainterp.history import ConstInt
+
+machine_ints = strategies.integers(min_value=-sys.maxint - 1,
+ max_value=sys.maxint)
+intboxes = strategies.builds(InputArgInt)
+intconsts = strategies.builds(ConstInt, machine_ints)
+boxes = intboxes | intconsts
+boxlists = strategies.lists(boxes, min_size=1).flatmap(
+ lambda cis: strategies.lists(strategies.sampled_from(cis)))
\ No newline at end of file
diff --git a/rpython/jit/metainterp/test/test_resume.py
b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -10,7 +10,7 @@
VArrayInfoNotClear, VStrPlainInfo, VStrConcatInfo, VStrSliceInfo,\
VUniPlainInfo, VUniConcatInfo, VUniSliceInfo, Snapshot, FrameInfo,\
capture_resumedata, ResumeDataLoopMemo, UNASSIGNEDVIRTUAL, INT,\
- annlowlevel, PENDINGFIELDSP, unpack_uint, TAG_CONST_OFFSET
+ annlowlevel, PENDINGFIELDSP, unpack_uint, TAG_CONST_OFFSET, TopSnapshot
from rpython.jit.metainterp.resumecode import unpack_numbering,\
create_numbering, NULL_NUMBER
@@ -22,8 +22,12 @@
from rpython.jit.codewriter import heaptracker, longlong
from rpython.jit.metainterp.resoperation import ResOperation, InputArgInt,\
InputArgRef, rop
+from rpython.jit.metainterp.test.strategies import boxlists
from rpython.rlib.debug import debug_start, debug_stop, debug_print,\
have_debug_prints
+from rpython.jit.metainterp import resumecode
+
+from hypothesis import given
class Storage:
rd_frame_info_list = None
@@ -278,9 +282,7 @@
assert bh.written_f == expected_f
-def Numbering(nums):
- numb = create_numbering(nums, 0)
- return numb
+Numbering = create_numbering
def tagconst(i):
return tag(i + TAG_CONST_OFFSET, TAGCONST)
@@ -610,7 +612,8 @@
assert unpack_uint(frame_info_list.packed_jitcode_pc) == (2, 15)
snapshot = storage.rd_snapshot
- assert snapshot.boxes == vrs + vbs # in the same list
+ assert snapshot.boxes == vrs
+ assert snapshot.vable_boxes == [b2, b1]
snapshot = snapshot.prev
assert snapshot.prev is fs[2].parent_resumedata_snapshot
@@ -904,9 +907,9 @@
env = [b1, c1, b2, b1, c2]
snap = Snapshot(None, env)
env1 = [c3, b3, b1, c1]
- snap1 = Snapshot(snap, env1)
+ snap1 = TopSnapshot(snap, env1, [])
env2 = [c3, b3, b1, c3]
- snap2 = Snapshot(snap, env2)
+ snap2 = TopSnapshot(snap, env2, [])
memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
frameinfo = FrameInfo(None, FakeJitCode("jitcode", 0), 0)
@@ -916,10 +919,11 @@
assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b3: tag(2, TAGBOX)}
- base = [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), tag(0, TAGBOX),
tag(2, TAGINT)]
+ base = [0, 0, tag(0, TAGBOX), tag(1, TAGINT),
+ tag(1, TAGBOX), tag(0, TAGBOX), tag(2, TAGINT)]
- assert unpack_numbering(numb) == [
- tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(1, TAGINT), 0,
0] + base
+ assert unpack_numbering(numb) == [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+ tag(0, TAGBOX), tag(1, TAGINT)] + base
numb2, liveboxes2, v = memo.number(FakeOptimizer(), snap2, frameinfo)
assert v == 0
@@ -927,11 +931,11 @@
assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b3: tag(2, TAGBOX)}
assert liveboxes2 is not liveboxes
- assert unpack_numbering(numb2) == [
- tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(3, TAGINT), 0, 0]
+ base
+ assert unpack_numbering(numb2) == [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+ tag(0, TAGBOX), tag(3, TAGINT)] + base
env3 = [c3, b3, b1, c3]
- snap3 = Snapshot(snap, env3)
+ snap3 = TopSnapshot(snap, env3, [])
class FakeVirtualInfo(info.AbstractInfo):
def __init__(self, virt):
@@ -946,13 +950,12 @@
assert v == 0
assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)}
- assert unpack_numbering(numb3) == [tag(3, TAGINT), tag(4, TAGINT),
- tag(0, TAGBOX),
- tag(3, TAGINT), 0, 0] + base
+ assert unpack_numbering(numb3) == [0, 2, tag(3, TAGINT), tag(4, TAGINT),
+ tag(0, TAGBOX), tag(3, TAGINT)] + base
# virtual
env4 = [c3, b4, b1, c3]
- snap4 = Snapshot(snap, env4)
+ snap4 = TopSnapshot(snap, env4, [])
b4.set_forwarded(FakeVirtualInfo(True))
numb4, liveboxes4, v = memo.number(FakeOptimizer(), snap4, frameinfo)
@@ -960,11 +963,11 @@
assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b4: tag(0, TAGVIRTUAL)}
- assert unpack_numbering(numb4) == [tag(3, TAGINT), tag(0, TAGVIRTUAL),
- tag(0, TAGBOX), tag(3, TAGINT), 0, 0] + base
+ assert unpack_numbering(numb4) == [0, 2, tag(3, TAGINT), tag(0,
TAGVIRTUAL),
+ tag(0, TAGBOX), tag(3, TAGINT)] + base
env5 = [b1, b4, b5]
- snap5 = Snapshot(snap4, env5)
+ snap5 = TopSnapshot(snap4, [], env5)
b4.set_forwarded(FakeVirtualInfo(True))
b5.set_forwarded(FakeVirtualInfo(True))
@@ -974,9 +977,30 @@
assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)}
- assert unpack_numbering(numb5) == [tag(0, TAGBOX), tag(0, TAGVIRTUAL),
- tag(1, TAGVIRTUAL), 2, 1] + unpack_numbering(numb4)
+ assert unpack_numbering(numb5) == [
+ 3, tag(0, TAGBOX), tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL),
+ 0,
+ 2, 1, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3,
TAGINT)
+ ] + base
+@given(boxlists)
+def test_ResumeDataLoopMemo_random(lst):
+ s = TopSnapshot(None, [], lst)
+ frameinfo = FrameInfo(None, FakeJitCode("foo", 0), 0)
+ memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
+ num, liveboxes, v = memo.number(FakeOptimizer(), s, frameinfo)
+ l = unpack_numbering(num)
+ assert l[-1] == 0
+ assert l[0] == len(lst)
+ for i, item in enumerate(lst):
+ v, tag = untag(l[i + 1])
+ if tag == TAGBOX:
+ assert l[i + 1] == liveboxes[item]
+ elif tag == TAGCONST:
+ assert memo.consts[v].getint() == item.getint()
+ elif tag == TAGINT:
+ assert v == item.getint()
+
def test_ResumeDataLoopMemo_number_boxes():
memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
b1, b2 = [InputArgInt(), InputArgInt()]
@@ -1060,10 +1084,11 @@
storage = Storage()
snapshot = Snapshot(None, [b1, ConstInt(1), b1, b2])
snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)])
- snapshot = Snapshot(snapshot, [b1, b2, b3])
- frameinfo = FrameInfo(FrameInfo(None, FakeJitCode("code1", 21), 22),
- FakeJitCode("code2", 31), 32)
- storage.rd_snapshot = snapshot
+ snapshot = Snapshot(snapshot, [b1, b2, b3])
+ top_snapshot = TopSnapshot(snapshot, [], [])
+ frameinfo = FrameInfo(FrameInfo(FrameInfo(None, FakeJitCode("code1", 21),
22),
+ FakeJitCode("code2", 31), 32), FakeJitCode("code3", 41), 42)
+ storage.rd_snapshot = top_snapshot
storage.rd_frame_info_list = frameinfo
return storage
@@ -1076,6 +1101,8 @@
assert storage.rd_snapshot is None
cpu = MyCPU([])
reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage, "deadframe")
+ reader.consume_vref_and_vable(None, None, None)
+ reader.cur_index += 2 # framestack
_next_section(reader, sys.maxint, 2**16, -65)
reader.cur_index += 2 # framestack
_next_section(reader, 2, 3)
@@ -1116,7 +1143,8 @@
class MyInfo:
@staticmethod
def enumerate_vars(callback_i, callback_r, callback_f, _, index):
- for tagged in self.numb.code:
+ while index < len(self.numb.code):
+ tagged, _ = resumecode.numb_next_item(self.numb, index)
_, tag = untag(tagged)
if tag == TAGVIRTUAL:
kind = REF
@@ -1131,6 +1159,13 @@
index = callback_f(index, index)
else:
assert 0
+ size, self.cur_index = resumecode.numb_next_item(self.numb, 0)
+ assert size == 0
+ size, self.cur_index = resumecode.numb_next_item(self.numb,
self.cur_index)
+ assert size == 0
+ pc, self.cur_index = resumecode.numb_next_item(self.numb,
self.cur_index)
+ jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb,
self.cur_index)
+
self._prepare_next_section(MyInfo())
return self.lst
diff --git a/rpython/jit/metainterp/test/test_resumecode.py
b/rpython/jit/metainterp/test/test_resumecode.py
--- a/rpython/jit/metainterp/test/test_resumecode.py
+++ b/rpython/jit/metainterp/test/test_resumecode.py
@@ -1,9 +1,12 @@
from rpython.jit.metainterp.resumecode import NUMBERING, NULL_NUMBER
from rpython.jit.metainterp.resumecode import create_numbering,\
- unpack_numbering, copy_from_list_to_numb
+ unpack_numbering
from rpython.rtyper.lltypesystem import lltype
+from hypothesis import strategies, given
+
+
def test_pack_unpack():
examples = [
[1, 2, 3, 4, 257, 10000, 13, 15],
@@ -12,5 +15,15 @@
[13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
]
for l in examples:
- n = create_numbering(l, 0)
+ n = create_numbering(l)
assert unpack_numbering(n) == l
+
+@given(strategies.lists(strategies.integers(-2**15, 2**15-1)))
+def test_roundtrip(l):
+ n = create_numbering(l)
+ assert unpack_numbering(n) == l
+
+@given(strategies.lists(strategies.integers(-2**15, 2**15-1)))
+def test_compressing(l):
+ n = create_numbering(l)
+ assert len(n.code) <= len(l) * 3
diff --git a/rpython/jit/metainterp/virtualizable.py
b/rpython/jit/metainterp/virtualizable.py
--- a/rpython/jit/metainterp/virtualizable.py
+++ b/rpython/jit/metainterp/virtualizable.py
@@ -142,6 +142,7 @@
item, index = numb_next_item(numb, index)
x = reader.load_value_of_type(ARRAYITEMTYPE, item)
setarrayitem(lst, j, x)
+ return index
def load_list_of_boxes(virtualizable, reader, vable_box, numb, index):
virtualizable = cast_gcref_to_vtype(virtualizable)
@@ -161,7 +162,7 @@
box = reader.decode_box_of_type(ARRAYITEMTYPE, item)
boxes.append(box)
boxes.append(vable_box)
- return boxes
+ return boxes, index
def check_boxes(virtualizable, boxes):
virtualizable = cast_gcref_to_vtype(virtualizable)
diff --git a/rpython/rlib/_os_support.py b/rpython/rlib/_os_support.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/_os_support.py
@@ -0,0 +1,109 @@
+import sys
+
+from rpython.annotator.model import s_Str0, s_Unicode0
+from rpython.rlib import rstring
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.lltypesystem import rffi
+
+
+_CYGWIN = sys.platform == 'cygwin'
+_WIN32 = sys.platform.startswith('win')
+UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
+_MACRO_ON_POSIX = True if not _WIN32 else None
+
+
+class StringTraits(object):
+ str = str
+ str0 = s_Str0
+ CHAR = rffi.CHAR
+ CCHARP = rffi.CCHARP
+ charp2str = staticmethod(rffi.charp2str)
+ charpsize2str = staticmethod(rffi.charpsize2str)
+ scoped_str2charp = staticmethod(rffi.scoped_str2charp)
+ str2charp = staticmethod(rffi.str2charp)
+ free_charp = staticmethod(rffi.free_charp)
+ scoped_alloc_buffer = staticmethod(rffi.scoped_alloc_buffer)
+
+ @staticmethod
+ @specialize.argtype(0)
+ def as_str(path):
+ assert path is not None
+ if isinstance(path, str):
+ return path
+ elif isinstance(path, unicode):
+ # This never happens in PyPy's Python interpreter!
+ # Only in raw RPython code that uses unicode strings.
+ # We implement python2 behavior: silently convert to ascii.
+ return path.encode('ascii')
+ else:
+ return path.as_bytes()
+
+ @staticmethod
+ @specialize.argtype(0)
+ def as_str0(path):
+ res = StringTraits.as_str(path)
+ rstring.check_str0(res)
+ return res
+
+
+class UnicodeTraits(object):
+ str = unicode
+ str0 = s_Unicode0
+ CHAR = rffi.WCHAR_T
+ CCHARP = rffi.CWCHARP
+ charp2str = staticmethod(rffi.wcharp2unicode)
+ charpsize2str = staticmethod(rffi.wcharpsize2unicode)
+ str2charp = staticmethod(rffi.unicode2wcharp)
+ scoped_str2charp = staticmethod(rffi.scoped_unicode2wcharp)
+ free_charp = staticmethod(rffi.free_wcharp)
+ scoped_alloc_buffer = staticmethod(rffi.scoped_alloc_unicodebuffer)
+
+ @staticmethod
+ @specialize.argtype(0)
+ def as_str(path):
+ assert path is not None
+ if isinstance(path, unicode):
+ return path
+ else:
+ return path.as_unicode()
+
+ @staticmethod
+ @specialize.argtype(0)
+ def as_str0(path):
+ res = UnicodeTraits.as_str(path)
+ rstring.check_str0(res)
+ return res
+
+
+string_traits = StringTraits()
+unicode_traits = UnicodeTraits()
+
+
+# Returns True when the unicode function should be called:
+# - on Windows
+# - if the path is Unicode.
+if _WIN32:
+ @specialize.argtype(0)
+ def _prefer_unicode(path):
+ assert path is not None
+ if isinstance(path, str):
+ return False
+ elif isinstance(path, unicode):
+ return True
+ else:
+ return path.is_unicode
+
+ @specialize.argtype(0)
+ def _preferred_traits(path):
+ if _prefer_unicode(path):
+ return unicode_traits
+ else:
+ return string_traits
+else:
+ @specialize.argtype(0)
+ def _prefer_unicode(path):
+ return False
+
+ @specialize.argtype(0)
+ def _preferred_traits(path):
+ return string_traits
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1,26 +1,22 @@
import os
import sys
import errno
+from rpython.annotator.model import s_Str0
from rpython.rtyper.lltypesystem.rffi import CConstant, CExternVariable, INT
from rpython.rtyper.lltypesystem import lltype, ll2ctypes, rffi
from rpython.rtyper.tool import rffi_platform
-from rpython.tool.sourcetools import func_renamer
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rlib.rarithmetic import intmask, widen
+from rpython.rlib import debug, jit, rstring, rthread, types
+from rpython.rlib._os_support import (
+ _CYGWIN, _MACRO_ON_POSIX, UNDERSCORE_ON_WIN32, _WIN32,
+ _prefer_unicode, _preferred_traits)
from rpython.rlib.objectmodel import (
specialize, enforceargs, register_replacement_for, NOT_CONSTANT)
+from rpython.rlib.rarithmetic import intmask, widen
from rpython.rlib.signature import signature
-from rpython.rlib import types
-from rpython.annotator.model import s_Str0, s_Unicode0
-from rpython.rlib import jit
+from rpython.tool.sourcetools import func_renamer
from rpython.translator.platform import platform
-from rpython.rlib import rstring
-from rpython.rlib import debug, rthread
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
-_WIN32 = sys.platform.startswith('win')
-_CYGWIN = sys.platform == 'cygwin'
-UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
-_MACRO_ON_POSIX = True if not _WIN32 else None
if _WIN32:
from rpython.rlib import rwin32
@@ -118,7 +114,6 @@
with the flag llexternal(..., save_err=rffi.RFFI_SAVE_ERRNO).
Functions without that flag don't change the saved errno.
"""
- from rpython.rlib import rthread
return intmask(rthread.tlfield_rpy_errno.getraw())
def set_saved_errno(errno):
@@ -129,7 +124,6 @@
zero; for that case, use llexternal(..., save_err=RFFI_ZERO_ERRNO_BEFORE)
and then you don't need set_saved_errno(0).
"""
- from rpython.rlib import rthread
rthread.tlfield_rpy_errno.setraw(rffi.cast(INT, errno))
def get_saved_alterrno():
@@ -138,7 +132,6 @@
with the flag llexternal(..., save_err=rffi.RFFI_SAVE_ERRNO |
rffl.RFFI_ALT_ERRNO).
Functions without that flag don't change the saved errno.
"""
- from rpython.rlib import rthread
return intmask(rthread.tlfield_alt_errno.getraw())
def set_saved_alterrno(errno):
@@ -149,7 +142,6 @@
zero; for that case, use llexternal(..., save_err=RFFI_ZERO_ERRNO_BEFORE)
and then you don't need set_saved_errno(0).
"""
- from rpython.rlib import rthread
rthread.tlfield_alt_errno.setraw(rffi.cast(INT, errno))
@@ -157,7 +149,6 @@
@specialize.call_location()
def _errno_before(save_err):
if save_err & rffi.RFFI_READSAVED_ERRNO:
- from rpython.rlib import rthread
if save_err & rffi.RFFI_ALT_ERRNO:
_set_errno(rthread.tlfield_alt_errno.getraw())
else:
@@ -165,7 +156,6 @@
elif save_err & rffi.RFFI_ZERO_ERRNO_BEFORE:
_set_errno(rffi.cast(rffi.INT, 0))
if _WIN32 and (save_err & rffi.RFFI_READSAVED_LASTERROR):
- from rpython.rlib import rthread, rwin32
if save_err & rffi.RFFI_ALT_ERRNO:
err = rthread.tlfield_alt_lasterror.getraw()
else:
@@ -179,7 +169,6 @@
def _errno_after(save_err):
if _WIN32:
if save_err & rffi.RFFI_SAVE_LASTERROR:
- from rpython.rlib import rthread, rwin32
err = rwin32._GetLastError()
# careful, setraw() overwrites GetLastError.
# We must read it first, before the errno handling.
@@ -188,14 +177,13 @@
else:
rthread.tlfield_rpy_lasterror.setraw(err)
elif save_err & rffi.RFFI_SAVE_WSALASTERROR:
- from rpython.rlib import rthread, _rsocket_rffi
+ from rpython.rlib import _rsocket_rffi
err = _rsocket_rffi._WSAGetLastError()
if save_err & rffi.RFFI_ALT_ERRNO:
rthread.tlfield_alt_lasterror.setraw(err)
else:
rthread.tlfield_rpy_lasterror.setraw(err)
if save_err & rffi.RFFI_SAVE_ERRNO:
- from rpython.rlib import rthread
if save_err & rffi.RFFI_ALT_ERRNO:
rthread.tlfield_alt_errno.setraw(_get_errno())
else:
@@ -341,118 +329,6 @@
rstring.check_str0(res)
return res
-
-class StringTraits:
- str = str
- str0 = s_Str0
- CHAR = rffi.CHAR
- CCHARP = rffi.CCHARP
- charp2str = staticmethod(rffi.charp2str)
- charpsize2str = staticmethod(rffi.charpsize2str)
- scoped_str2charp = staticmethod(rffi.scoped_str2charp)
- str2charp = staticmethod(rffi.str2charp)
- free_charp = staticmethod(rffi.free_charp)
- scoped_alloc_buffer = staticmethod(rffi.scoped_alloc_buffer)
-
- @staticmethod
- def posix_function_name(name):
- return UNDERSCORE_ON_WIN32 + name
-
- @staticmethod
- def ll_os_name(name):
- return 'll_os.ll_os_' + name
-
- @staticmethod
- @specialize.argtype(0)
- def as_str(path):
- assert path is not None
- if isinstance(path, str):
- return path
- elif isinstance(path, unicode):
- # This never happens in PyPy's Python interpreter!
- # Only in raw RPython code that uses unicode strings.
- # We implement python2 behavior: silently convert to ascii.
- return path.encode('ascii')
- else:
- return path.as_bytes()
-
- @staticmethod
- @specialize.argtype(0)
- def as_str0(path):
- res = StringTraits.as_str(path)
- rstring.check_str0(res)
- return res
-
-
-class UnicodeTraits:
- str = unicode
- str0 = s_Unicode0
- CHAR = rffi.WCHAR_T
- CCHARP = rffi.CWCHARP
- charp2str = staticmethod(rffi.wcharp2unicode)
- charpsize2str = staticmethod(rffi.wcharpsize2unicode)
- str2charp = staticmethod(rffi.unicode2wcharp)
- scoped_str2charp = staticmethod(rffi.scoped_unicode2wcharp)
- free_charp = staticmethod(rffi.free_wcharp)
- scoped_alloc_buffer = staticmethod(rffi.scoped_alloc_unicodebuffer)
-
- @staticmethod
- def posix_function_name(name):
- return UNDERSCORE_ON_WIN32 + 'w' + name
-
- @staticmethod
- @specialize.argtype(0)
- def ll_os_name(name):
- return 'll_os.ll_os_w' + name
-
- @staticmethod
- @specialize.argtype(0)
- def as_str(path):
- assert path is not None
- if isinstance(path, unicode):
- return path
- else:
- return path.as_unicode()
-
- @staticmethod
- @specialize.argtype(0)
- def as_str0(path):
- res = UnicodeTraits.as_str(path)
- rstring.check_str0(res)
- return res
-
-
-# Returns True when the unicode function should be called:
-# - on Windows
-# - if the path is Unicode.
-unicode_traits = UnicodeTraits()
-string_traits = StringTraits()
-if _WIN32:
- @specialize.argtype(0)
- def _prefer_unicode(path):
- assert path is not None
- if isinstance(path, str):
- return False
- elif isinstance(path, unicode):
- return True
- else:
- return path.is_unicode
-
- @specialize.argtype(0)
- def _preferred_traits(path):
- if _prefer_unicode(path):
- return unicode_traits
- else:
- return string_traits
-else:
- @specialize.argtype(0)
- def _prefer_unicode(path):
- return False
-
- @specialize.argtype(0)
- def _preferred_traits(path):
- return string_traits
-
@specialize.argtype(0, 1)
def putenv(name, value):
os.environ[_as_bytes(name)] = _as_bytes(value)
diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py
--- a/rpython/rlib/rposix_environ.py
+++ b/rpython/rlib/rposix_environ.py
@@ -1,8 +1,9 @@
import os
import sys
from rpython.annotator import model as annmodel
+from rpython.rlib._os_support import _WIN32, StringTraits, UnicodeTraits
from rpython.rlib.objectmodel import enforceargs
-from rpython.rlib.rposix import _WIN32, StringTraits, UnicodeTraits
+# importing rposix here creates a cycle on Windows
from rpython.rtyper.controllerentry import Controller
from rpython.rtyper.extfunc import register_external
from rpython.rtyper.lltypesystem import rffi, lltype
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -16,13 +16,13 @@
from rpython.rtyper.rint import IntegerRepr
from rpython.rtyper.error import TyperError
+from rpython.rlib._os_support import _preferred_traits, string_traits
from rpython.rlib.objectmodel import specialize
from rpython.rtyper.lltypesystem import lltype, rffi
from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.rlib.rarithmetic import intmask
from rpython.rlib.rposix import (
- replace_os_function, handle_posix_error, _as_bytes0,
- _preferred_traits, string_traits)
+ replace_os_function, handle_posix_error, _as_bytes0)
_WIN32 = sys.platform.startswith('win')
_LINUX = sys.platform.startswith('linux')
diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py
b/rpython/rtyper/lltypesystem/ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/ll2ctypes.py
@@ -426,7 +426,12 @@
else:
n = None
cstruct = cls._malloc(n)
- add_storage(container, _struct_mixin, ctypes.pointer(cstruct))
+
+ if isinstance(container, lltype._fixedsizearray):
+ cls_mixin = _fixedsizedarray_mixin
+ else:
+ cls_mixin = _struct_mixin
+ add_storage(container, cls_mixin, ctypes.pointer(cstruct))
if delayed_converters is None:
delayed_converters_was_None = True
@@ -506,7 +511,11 @@
def struct_use_ctypes_storage(container, ctypes_storage):
STRUCT = container._TYPE
assert isinstance(STRUCT, lltype.Struct)
- add_storage(container, _struct_mixin, ctypes_storage)
+ if isinstance(container, lltype._fixedsizearray):
+ cls_mixin = _fixedsizedarray_mixin
+ else:
+ cls_mixin = _struct_mixin
+ add_storage(container, cls_mixin, ctypes_storage)
remove_regular_struct_content(container)
for field_name in STRUCT._names:
FIELDTYPE = getattr(STRUCT, field_name)
@@ -533,7 +542,7 @@
# Ctypes-aware subclasses of the _parentable classes
ALLOCATED = {} # mapping {address: _container}
-DEBUG_ALLOCATED = True
+DEBUG_ALLOCATED = False
def get_common_subclass(cls1, cls2, cache={}):
"""Return a unique subclass with (cls1, cls2) as bases."""
@@ -655,11 +664,44 @@
cobj = lltype2ctypes(value)
setattr(self._storage.contents, field_name, cobj)
+class _fixedsizedarray_mixin(_parentable_mixin):
+ """Mixin added to _fixedsizearray containers when they become
ctypes-based."""
+ __slots__ = ()
+
+ def __getattr__(self, field_name):
+ if hasattr(self, '_items'):
+ obj = lltype._fixedsizearray.__getattr__.im_func(self, field_name)
+ return obj
+ else:
+ cobj = getattr(self._storage.contents, field_name)
+ T = getattr(self._TYPE, field_name)
+ return ctypes2lltype(T, cobj)
+
+ def __setattr__(self, field_name, value):
+ if field_name.startswith('_'):
+ object.__setattr__(self, field_name, value) # '_xxx' attributes
+ else:
+ cobj = lltype2ctypes(value)
+ if hasattr(self, '_items'):
+ lltype._fixedsizearray.__setattr__.im_func(self, field_name,
cobj)
+ else:
+ setattr(self._storage.contents, field_name, cobj)
+
+
def getitem(self, index, uninitialized_ok=False):
- return getattr(self, "item%s" % index)
+ if hasattr(self, '_items'):
+ obj = lltype._fixedsizearray.getitem.im_func(self,
+ index, uninitialized_ok=uninitialized_ok)
+ return obj
+ else:
+ return getattr(self, 'item%d' % index)
def setitem(self, index, value):
- setattr(self, "item%s" % index, value)
+ cobj = lltype2ctypes(value)
+ if hasattr(self, '_items'):
+ lltype._fixedsizearray.setitem.im_func(self, index, value)
+ else:
+ setattr(self, 'item%d' % index, cobj)
class _array_mixin(_parentable_mixin):
"""Mixin added to _array containers when they become ctypes-based."""
diff --git a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py
b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py
@@ -1461,3 +1461,20 @@
assert a[3].a == 17
#lltype.free(a, flavor='raw')
py.test.skip("free() not working correctly here...")
+
+ def test_fixedsizedarray_to_ctypes(self):
+ T = lltype.Ptr(rffi.CFixedArray(rffi.INT, 1))
+ inst = lltype.malloc(T.TO, flavor='raw')
+ inst[0] = rffi.cast(rffi.INT, 42)
+ assert inst[0] == 42
+ cinst = lltype2ctypes(inst)
+ assert rffi.cast(lltype.Signed, inst[0]) == 42
+ assert cinst.contents.item0 == 42
+ lltype.free(inst, flavor='raw')
+
+ def test_fixedsizedarray_to_ctypes(self):
+ T = lltype.Ptr(rffi.CFixedArray(rffi.CHAR, 123))
+ inst = lltype.malloc(T.TO, flavor='raw', zero=True)
+ cinst = lltype2ctypes(inst)
+ assert cinst.contents.item0 == 0
+ lltype.free(inst, flavor='raw')
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
