Author: Armin Rigo <ar...@tunes.org>
Branch:
Changeset: r83961:b985ddd2357b
Date: 2016-04-27 09:16 +0200
http://bitbucket.org/pypy/pypy/changeset/b985ddd2357b/
Log: hg merge bitstring
Use bitstrings to compress the lists of read or written descrs that
we attach to EffectInfo. It is not necessarily more compact (if
there are a lot of tiny lists), but it fixes two problems:
* optimizeopt/heap.py: we no longer have the potential problem of
having to walk some very large lists of descrs. Now we walk the
descrs that are cached at this point, and check if each one is in
the bitstring.
* backendopt/writeanalyze.py: as a result we can get rid of the
arbitrary limit on the size of the lists, which got in the way
recently.
diff --git a/rpython/annotator/test/test_annrpython.py
b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4577,6 +4577,13 @@
with py.test.raises(AnnotatorError):
a.build_types(f, [float])
+ def test_Ellipsis_not_rpython(self):
+ def f():
+ return Ellipsis
+ a = self.RPythonAnnotator()
+ e = py.test.raises(Exception, a.build_types, f, [])
+ assert str(e.value) == "Don't know how to represent Ellipsis"
+
def g(n):
return [0, 1, 2, n]
diff --git a/rpython/jit/backend/llgraph/runner.py
b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -479,6 +479,9 @@
all_descrs.append(v)
return all_descrs
+ def fetch_all_descrs(self):
+ return self.descrs.values()
+
def calldescrof(self, FUNC, ARGS, RESULT, effect_info):
key = ('call', getkind(RESULT),
tuple([getkind(A) for A in ARGS]),
diff --git a/rpython/jit/codewriter/effectinfo.py
b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -1,7 +1,9 @@
+import sys
from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem
from rpython.rtyper.rclass import OBJECT
from rpython.rtyper.lltypesystem import lltype, llmemory
from rpython.translator.backendopt.graphanalyze import BoolGraphAnalyzer
+from rpython.tool.algo import bitstring
class EffectInfo(object):
@@ -110,12 +112,20 @@
can_invalidate=False,
call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
extradescrs=None):
- key = (frozenset_or_none(readonly_descrs_fields),
- frozenset_or_none(readonly_descrs_arrays),
- frozenset_or_none(readonly_descrs_interiorfields),
- frozenset_or_none(write_descrs_fields),
- frozenset_or_none(write_descrs_arrays),
- frozenset_or_none(write_descrs_interiorfields),
+ readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
+ readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
+ readonly_descrs_interiorfields = frozenset_or_none(
+ readonly_descrs_interiorfields)
+ write_descrs_fields = frozenset_or_none(write_descrs_fields)
+ write_descrs_arrays = frozenset_or_none(write_descrs_arrays)
+ write_descrs_interiorfields = frozenset_or_none(
+ write_descrs_interiorfields)
+ key = (readonly_descrs_fields,
+ readonly_descrs_arrays,
+ readonly_descrs_interiorfields,
+ write_descrs_fields,
+ write_descrs_arrays,
+ write_descrs_interiorfields,
extraeffect,
oopspecindex,
can_invalidate)
@@ -139,22 +149,34 @@
assert write_descrs_arrays is not None
assert write_descrs_interiorfields is not None
result = object.__new__(cls)
- result.readonly_descrs_fields = readonly_descrs_fields
- result.readonly_descrs_arrays = readonly_descrs_arrays
- result.readonly_descrs_interiorfields = readonly_descrs_interiorfields
+ # the frozensets "._readonly_xxx" and "._write_xxx" should not be
+ # translated.
+ result._readonly_descrs_fields = readonly_descrs_fields
+ result._readonly_descrs_arrays = readonly_descrs_arrays
+ result._readonly_descrs_interiorfields = readonly_descrs_interiorfields
if extraeffect == EffectInfo.EF_LOOPINVARIANT or \
extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \
extraeffect == EffectInfo.EF_ELIDABLE_OR_MEMORYERROR or \
extraeffect == EffectInfo.EF_ELIDABLE_CAN_RAISE:
# Ignore the writes. Note that this ignores also writes with
# no corresponding reads (rarely the case, but possible).
- result.write_descrs_fields = []
- result.write_descrs_arrays = []
- result.write_descrs_interiorfields = []
+ result._write_descrs_fields = frozenset()
+ result._write_descrs_arrays = frozenset()
+ result._write_descrs_interiorfields = frozenset()
else:
- result.write_descrs_fields = write_descrs_fields
- result.write_descrs_arrays = write_descrs_arrays
- result.write_descrs_interiorfields = write_descrs_interiorfields
+ result._write_descrs_fields = write_descrs_fields
+ result._write_descrs_arrays = write_descrs_arrays
+ result._write_descrs_interiorfields = write_descrs_interiorfields
+ # initialized later, in compute_bitstrings()
+ # (the goal of this is to make sure we don't build new EffectInfo
+ # instances after compute_bitstrings() is called)
+ result.bitstring_readonly_descrs_fields = Ellipsis
+ result.bitstring_readonly_descrs_arrays = Ellipsis
+ result.bitstring_readonly_descrs_interiorfields = Ellipsis
+ result.bitstring_write_descrs_fields = Ellipsis
+ result.bitstring_write_descrs_arrays = Ellipsis
+ result.bitstring_write_descrs_interiorfields = Ellipsis
+ #
result.extraeffect = extraeffect
result.can_invalidate = can_invalidate
result.oopspecindex = oopspecindex
@@ -162,9 +184,38 @@
result.call_release_gil_target = call_release_gil_target
if result.check_can_raise(ignore_memoryerror=True):
assert oopspecindex in cls._OS_CANRAISE
+
+ if (result._write_descrs_arrays is not None and
+ len(result._write_descrs_arrays) == 1):
+ # this is used only for ARRAYCOPY operations
+ [result.single_write_descr_array] = result._write_descrs_arrays
+ else:
+ result.single_write_descr_array = None
+
cls._cache[key] = result
return result
+ def check_readonly_descr_field(self, fielddescr):
+ return bitstring.bitcheck(self.bitstring_readonly_descrs_fields,
+ fielddescr.ei_index)
+ def check_write_descr_field(self, fielddescr):
+ return bitstring.bitcheck(self.bitstring_write_descrs_fields,
+ fielddescr.ei_index)
+ def check_readonly_descr_array(self, arraydescr):
+ return bitstring.bitcheck(self.bitstring_readonly_descrs_arrays,
+ arraydescr.ei_index)
+ def check_write_descr_array(self, arraydescr):
+ return bitstring.bitcheck(self.bitstring_write_descrs_arrays,
+ arraydescr.ei_index)
+ def check_readonly_descr_interiorfield(self, interiorfielddescr):
+ # NOTE: this is not used so far
+ return
bitstring.bitcheck(self.bitstring_readonly_descrs_interiorfields,
+ interiorfielddescr.ei_index)
+ def check_write_descr_interiorfield(self, interiorfielddescr):
+ # NOTE: this is not used so far
+ return bitstring.bitcheck(self.bitstring_write_descrs_interiorfields,
+ interiorfielddescr.ei_index)
+
def check_can_raise(self, ignore_memoryerror=False):
if ignore_memoryerror:
return self.extraeffect > self.EF_ELIDABLE_OR_MEMORYERROR
@@ -382,3 +433,88 @@
assert funcptr
return funcptr
funcptr_for_oopspec._annspecialcase_ = 'specialize:arg(1)'
+
+# ____________________________________________________________
+
+def compute_bitstrings(all_descrs):
+ # Compute the bitstrings in the EffectInfo,
+ # bitstring_{readonly,write}_descrs_{fieldd,arrays,interiordescrs},
+ # and for each FieldDescrs and ArrayDescrs compute 'ei_index'.
+ # Each bit in the bitstrings says whether this Descr is present in
+ # this EffectInfo or not. We try to share the value of 'ei_index'
+ # across multiple Descrs if they always give the same answer (in
+ # PyPy, it reduces the length of the bitstrings from 4000+ to
+ # 373).
+ from rpython.jit.codewriter.policy import log
+
+ log("compute_bitstrings:")
+ effectinfos = []
+ descrs = {'fields': set(), 'arrays': set(), 'interiorfields': set()}
+ for descr in all_descrs:
+ if hasattr(descr, 'get_extra_info'):
+ ei = descr.get_extra_info()
+ if ei is None:
+ continue
+ if ei._readonly_descrs_fields is None:
+ for key in descrs:
+ assert getattr(ei, '_readonly_descrs_' + key) is None
+ assert getattr(ei, '_write_descrs_' + key) is None
+ setattr(ei, 'bitstring_readonly_descrs_' + key, None)
+ setattr(ei, 'bitstring_write_descrs_' + key, None)
+ else:
+ effectinfos.append(ei)
+ for key in descrs:
+ descrs[key].update(getattr(ei, '_readonly_descrs_' + key))
+ descrs[key].update(getattr(ei, '_write_descrs_' + key))
+ else:
+ descr.ei_index = sys.maxint
+ log(" %d effectinfos:" % (len(effectinfos),))
+ for key in sorted(descrs):
+ log(" %d descrs for %s" % (len(descrs[key]), key))
+
+ seen = set()
+ for key in descrs:
+ all_sets = []
+ for descr in descrs[key]:
+ eisetr = [ei for ei in effectinfos
+ if descr in getattr(ei, '_readonly_descrs_' + key)]
+ eisetw = [ei for ei in effectinfos
+ if descr in getattr(ei, '_write_descrs_' + key)]
+ # these are the set of all ei such that this descr is in
+ # ei._readonly_descrs or ei._write_descrs
+ eisetr = frozenset(eisetr)
+ eisetw = frozenset(eisetw)
+ all_sets.append((descr, eisetr, eisetw))
+
+ # heuristic to reduce the total size of the bitstrings: start with
+ # numbering the descrs that are seen in many EffectInfos. If instead,
+ # by lack of chance, such a descr had a high number, then all these
+ # EffectInfos' bitstrings would need to store the same high number.
+ def size_of_both_sets((d, r, w)):
+ return len(r) + len(w)
+ all_sets.sort(key=size_of_both_sets, reverse=True)
+
+ mapping = {}
+ for (descr, eisetr, eisetw) in all_sets:
+ assert descr.ei_index == sys.maxint # not modified yet
+ descr.ei_index = mapping.setdefault((eisetr, eisetw), len(mapping))
+
+ for ei in effectinfos:
+ bitstrr = [descr.ei_index
+ for descr in getattr(ei, '_readonly_descrs_' + key)]
+ bitstrw = [descr.ei_index
+ for descr in getattr(ei, '_write_descrs_' + key)]
+ assert sys.maxint not in bitstrr
+ assert sys.maxint not in bitstrw
+ bitstrr = bitstring.make_bitstring(bitstrr)
+ bitstrw = bitstring.make_bitstring(bitstrw)
+ setattr(ei, 'bitstring_readonly_descrs_' + key, bitstrr)
+ setattr(ei, 'bitstring_write_descrs_' + key, bitstrw)
+ seen.add(bitstrr)
+ seen.add(bitstrw)
+
+ if seen:
+ mean_length = float(sum(len(x) for x in seen)) / len(seen)
+ max_length = max(len(x) for x in seen)
+ log("-> %d bitstrings, mean length %.1f, max length %d" % (
+ len(seen), mean_length, max_length))
diff --git a/rpython/jit/codewriter/test/test_effectinfo.py
b/rpython/jit/codewriter/test/test_effectinfo.py
--- a/rpython/jit/codewriter/test/test_effectinfo.py
+++ b/rpython/jit/codewriter/test/test_effectinfo.py
@@ -1,11 +1,12 @@
-import pytest
+import pytest, sys
from rpython.jit.codewriter.effectinfo import (effectinfo_from_writeanalyze,
- EffectInfo, VirtualizableAnalyzer)
+ EffectInfo, VirtualizableAnalyzer, compute_bitstrings)
from rpython.rlib import jit
from rpython.rtyper.lltypesystem import lltype
from rpython.rtyper.rclass import OBJECT
from rpython.translator.translator import TranslationContext, graphof
+from rpython.tool.algo.bitstring import bitcheck
class FakeCPU(object):
@@ -29,37 +30,39 @@
S = lltype.GcStruct("S", ("a", lltype.Signed))
effects = frozenset([("readstruct", lltype.Ptr(S), "a")])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert list(effectinfo.readonly_descrs_fields) == [('fielddescr', S, "a")]
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert list(effectinfo._readonly_descrs_fields) == [('fielddescr', S, "a")]
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
+ assert effectinfo.single_write_descr_array is None
def test_include_write_field():
S = lltype.GcStruct("S", ("a", lltype.Signed))
effects = frozenset([("struct", lltype.Ptr(S), "a")])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")]
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert list(effectinfo._write_descrs_fields) == [('fielddescr', S, "a")]
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_arrays
def test_include_read_array():
A = lltype.GcArray(lltype.Signed)
effects = frozenset([("readarray", lltype.Ptr(A))])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert not effectinfo.readonly_descrs_fields
- assert list(effectinfo.readonly_descrs_arrays) == [('arraydescr', A)]
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert list(effectinfo._readonly_descrs_arrays) == [('arraydescr', A)]
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
def test_include_write_array():
A = lltype.GcArray(lltype.Signed)
effects = frozenset([("array", lltype.Ptr(A))])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_fields
- assert list(effectinfo.write_descrs_arrays) == [('arraydescr', A)]
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_fields
+ assert list(effectinfo._write_descrs_arrays) == [('arraydescr', A)]
+ assert effectinfo.single_write_descr_array == ('arraydescr', A)
def test_dont_include_read_and_write_field():
@@ -67,9 +70,9 @@
effects = frozenset([("readstruct", lltype.Ptr(S), "a"),
("struct", lltype.Ptr(S), "a")])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert not effectinfo.readonly_descrs_fields
- assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")]
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert list(effectinfo._write_descrs_fields) == [('fielddescr', S, "a")]
+ assert not effectinfo._write_descrs_arrays
def test_dont_include_read_and_write_array():
@@ -77,34 +80,34 @@
effects = frozenset([("readarray", lltype.Ptr(A)),
("array", lltype.Ptr(A))])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.readonly_descrs_arrays
- assert not effectinfo.write_descrs_fields
- assert list(effectinfo.write_descrs_arrays) == [('arraydescr', A)]
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._readonly_descrs_arrays
+ assert not effectinfo._write_descrs_fields
+ assert list(effectinfo._write_descrs_arrays) == [('arraydescr', A)]
def test_filter_out_typeptr():
effects = frozenset([("struct", lltype.Ptr(OBJECT), "typeptr")])
effectinfo = effectinfo_from_writeanalyze(effects, None)
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
def test_filter_out_array_of_void():
effects = frozenset([("array", lltype.Ptr(lltype.GcArray(lltype.Void)))])
effectinfo = effectinfo_from_writeanalyze(effects, None)
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
def test_filter_out_struct_with_void():
effects = frozenset([("struct", lltype.Ptr(lltype.GcStruct("x", ("a",
lltype.Void))), "a")])
effectinfo = effectinfo_from_writeanalyze(effects, None)
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
class TestVirtualizableAnalyzer(object):
@@ -138,3 +141,64 @@
res = self.analyze(entry, [int])
assert not res
+
+
+def test_compute_bitstrings():
+ class FDescr:
+ pass
+ class ADescr:
+ pass
+ class CDescr:
+ def __init__(self, ei):
+ self._ei = ei
+ def get_extra_info(self):
+ return self._ei
+
+ f1descr = FDescr()
+ f2descr = FDescr()
+ f3descr = FDescr()
+ a1descr = ADescr()
+ a2descr = ADescr()
+
+ ei1 = EffectInfo(None, None, None, None, None, None,
+ EffectInfo.EF_RANDOM_EFFECTS)
+ ei2 = EffectInfo([f1descr], [], [], [], [], [])
+ ei3 = EffectInfo([f1descr], [a1descr, a2descr], [], [f2descr], [], [])
+
+ compute_bitstrings([CDescr(ei1), CDescr(ei2), CDescr(ei3),
+ f1descr, f2descr, f3descr, a1descr, a2descr])
+
+ assert f1descr.ei_index in (0, 1)
+ assert f2descr.ei_index == 1 - f1descr.ei_index
+ assert f3descr.ei_index == sys.maxint
+ assert a1descr.ei_index == 0
+ assert a2descr.ei_index == 0
+
+ assert ei1.bitstring_readonly_descrs_fields is None
+ assert ei1.bitstring_readonly_descrs_arrays is None
+ assert ei1.bitstring_write_descrs_fields is None
+
+ def expand(bitstr):
+ return [n for n in range(10) if bitcheck(bitstr, n)]
+
+ assert expand(ei2.bitstring_readonly_descrs_fields) == [f1descr.ei_index]
+ assert expand(ei2.bitstring_write_descrs_fields) == []
+ assert expand(ei2.bitstring_readonly_descrs_arrays) == []
+ assert expand(ei2.bitstring_write_descrs_arrays) == []
+
+ assert expand(ei3.bitstring_readonly_descrs_fields) == [f1descr.ei_index]
+ assert expand(ei3.bitstring_write_descrs_fields) == [f2descr.ei_index]
+ assert expand(ei3.bitstring_readonly_descrs_arrays) == [0] #a1descr,a2descr
+ assert expand(ei3.bitstring_write_descrs_arrays) == []
+
+ for ei in [ei2, ei3]:
+ for fdescr in [f1descr, f2descr]:
+ assert ei.check_readonly_descr_field(fdescr) == (
+ fdescr in ei._readonly_descrs_fields)
+ assert ei.check_write_descr_field(fdescr) == (
+ fdescr in ei._write_descrs_fields)
+ for adescr in [a1descr, a2descr]:
+ assert ei.check_readonly_descr_array(adescr) == (
+ adescr in ei._readonly_descrs_arrays)
+ assert ei.check_write_descr_array(adescr) == (
+ adescr in ei._write_descrs_arrays)
diff --git a/rpython/jit/metainterp/heapcache.py
b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -209,7 +209,7 @@
isinstance(argboxes[3], ConstInt) and
isinstance(argboxes[4], ConstInt) and
isinstance(argboxes[5], ConstInt) and
- len(descr.get_extra_info().write_descrs_arrays) == 1):
+ descr.get_extra_info().single_write_descr_array is not None):
# ARRAYCOPY with constant starts and constant length doesn't escape
# its argument
# XXX really?
@@ -299,9 +299,9 @@
isinstance(argboxes[3], ConstInt) and
isinstance(argboxes[4], ConstInt) and
isinstance(argboxes[5], ConstInt) and
- len(effectinfo.write_descrs_arrays) == 1
+ effectinfo.single_write_descr_array is not None
):
- descr = effectinfo.write_descrs_arrays[0]
+ descr = effectinfo.single_write_descr_array
cache = self.heap_array_cache.get(descr, None)
srcstart = argboxes[3].getint()
dststart = argboxes[4].getint()
@@ -328,10 +328,10 @@
idx_cache._clear_cache_on_write(seen_allocation_of_target)
return
elif (
- len(effectinfo.write_descrs_arrays) == 1
+ effectinfo.single_write_descr_array is not None
):
# Fish the descr out of the effectinfo
- cache =
self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
+ cache =
self.heap_array_cache.get(effectinfo.single_write_descr_array, None)
if cache is not None:
for idx, cache in cache.iteritems():
cache._clear_cache_on_write(seen_allocation_of_target)
diff --git a/rpython/jit/metainterp/history.py
b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -1,3 +1,4 @@
+import sys
from rpython.rtyper.extregistry import ExtRegistryEntry
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
from rpython.rlib.objectmodel import we_are_translated, Symbolic
@@ -87,9 +88,10 @@
class AbstractDescr(AbstractValue):
- __slots__ = ('descr_index',)
+ __slots__ = ('descr_index', 'ei_index')
llopaque = True
descr_index = -1
+ ei_index = sys.maxint
def repr_of_descr(self):
return '%r' % (self,)
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py
b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -432,28 +432,35 @@
optimize_GUARD_EXCEPTION = optimize_GUARD_NO_EXCEPTION
def force_from_effectinfo(self, effectinfo):
- # XXX we can get the wrong complexity here, if the lists
- # XXX stored on effectinfo are large
- for fielddescr in effectinfo.readonly_descrs_fields:
- self.force_lazy_set(fielddescr)
- for arraydescr in effectinfo.readonly_descrs_arrays:
- self.force_lazy_setarrayitem(arraydescr)
- for fielddescr in effectinfo.write_descrs_fields:
- if fielddescr.is_always_pure():
- continue
- try:
- del self.cached_dict_reads[fielddescr]
- except KeyError:
- pass
- self.force_lazy_set(fielddescr, can_cache=False)
- for arraydescr in effectinfo.write_descrs_arrays:
- self.force_lazy_setarrayitem(arraydescr, can_cache=False)
- if arraydescr in self.corresponding_array_descrs:
- dictdescr = self.corresponding_array_descrs.pop(arraydescr)
+ # Note: this version of the code handles effectively
+ # effectinfos that store arbitrarily many descrs, by looping
+ # on self.cached_{fields, arrayitems} and looking them up in
+ # the bitstrings stored in the effectinfo.
+ for fielddescr, cf in self.cached_fields.items():
+ if effectinfo.check_readonly_descr_field(fielddescr):
+ cf.force_lazy_set(self, fielddescr)
+ if effectinfo.check_write_descr_field(fielddescr):
+ if fielddescr.is_always_pure():
+ continue
+ try:
+ del self.cached_dict_reads[fielddescr]
+ except KeyError:
+ pass
+ cf.force_lazy_set(self, fielddescr, can_cache=False)
+ #
+ for arraydescr, submap in self.cached_arrayitems.items():
+ if effectinfo.check_readonly_descr_array(arraydescr):
+ self.force_lazy_setarrayitem_submap(submap)
+ if effectinfo.check_write_descr_array(arraydescr):
+ self.force_lazy_setarrayitem_submap(submap, can_cache=False)
+ #
+ for arraydescr, dictdescr in self.corresponding_array_descrs.items():
+ if effectinfo.check_write_descr_array(arraydescr):
try:
del self.cached_dict_reads[dictdescr]
except KeyError:
pass # someone did it already
+ #
if effectinfo.check_forces_virtual_or_virtualizable():
vrefinfo = self.optimizer.metainterp_sd.virtualref_info
self.force_lazy_set(vrefinfo.descr_forced)
@@ -476,6 +483,10 @@
if indexb is None or indexb.contains(idx):
cf.force_lazy_set(self, None, can_cache)
+ def force_lazy_setarrayitem_submap(self, submap, can_cache=True):
+ for cf in submap.itervalues():
+ cf.force_lazy_set(self, None, can_cache)
+
def force_all_lazy_sets(self):
items = self.cached_fields.items()
if not we_are_translated():
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py
b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -620,10 +620,10 @@
and length and ((dest_info and dest_info.is_virtual()) or
length.getint() <= 8) and
((source_info and source_info.is_virtual()) or length.getint() <=
8)
- and len(extrainfo.write_descrs_arrays) == 1): # <-sanity check
+ and extrainfo.single_write_descr_array is not None): #<-sanity
check
source_start = source_start_box.getint()
dest_start = dest_start_box.getint()
- arraydescr = extrainfo.write_descrs_arrays[0]
+ arraydescr = extrainfo.single_write_descr_array
if arraydescr.is_array_of_structs():
return False # not supported right now
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py
b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -10,7 +10,7 @@
from rpython.jit.metainterp.history import (TreeLoop, AbstractDescr,
JitCellToken, TargetToken)
from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
-from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter.effectinfo import EffectInfo, compute_bitstrings
from rpython.jit.metainterp.logger import LogOperations
from rpython.jit.tool.oparser import OpParser, pure_parse,
convert_loop_to_trace
from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
@@ -530,6 +530,7 @@
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
+ compute_bitstrings(self.cpu.fetch_all_descrs())
#
compile_data.enable_opts = self.enable_opts
state = optimize_trace(metainterp_sd, None, compile_data)
diff --git a/rpython/jit/metainterp/pyjitpl.py
b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1838,7 +1838,11 @@
self.cpu.propagate_exception_descr = exc_descr
#
self.globaldata = MetaInterpGlobalData(self)
+
+ def finish_setup_descrs(self):
+ from rpython.jit.codewriter import effectinfo
self.all_descrs = self.cpu.setup_descrs()
+ effectinfo.compute_bitstrings(self.all_descrs)
def _setup_once(self):
"""Runtime setup needed by the various components of the JIT."""
diff --git a/rpython/jit/metainterp/test/support.py
b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -132,6 +132,7 @@
metainterp_sd = pyjitpl.MetaInterpStaticData(cw.cpu, opt)
stats.metainterp_sd = metainterp_sd
metainterp_sd.finish_setup(cw)
+ metainterp_sd.finish_setup_descrs()
[jitdriver_sd] = metainterp_sd.jitdrivers_sd
metainterp = pyjitpl.MetaInterp(metainterp_sd, jitdriver_sd)
diff --git a/rpython/jit/metainterp/test/test_heapcache.py
b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -27,8 +27,12 @@
def __init__(self, extraeffect, oopspecindex, write_descrs_fields,
write_descrs_arrays):
self.extraeffect = extraeffect
self.oopspecindex = oopspecindex
- self.write_descrs_fields = write_descrs_fields
- self.write_descrs_arrays = write_descrs_arrays
+ self._write_descrs_fields = write_descrs_fields
+ self._write_descrs_arrays = write_descrs_arrays
+ if len(write_descrs_arrays) == 1:
+ [self.single_write_descr_array] = write_descrs_arrays
+ else:
+ self.single_write_descr_array = None
def has_random_effects(self):
return self.extraeffect == self.EF_RANDOM_EFFECTS
@@ -37,14 +41,14 @@
def __init__(self, extraeffect, oopspecindex=None, write_descrs_fields=[],
write_descrs_arrays=[]):
self.extraeffect = extraeffect
self.oopspecindex = oopspecindex
- self.write_descrs_fields = write_descrs_fields
- self.write_descrs_arrays = write_descrs_arrays
+ self.__write_descrs_fields = write_descrs_fields
+ self.__write_descrs_arrays = write_descrs_arrays
def get_extra_info(self):
return FakeEffectinfo(
self.extraeffect, self.oopspecindex,
- write_descrs_fields=self.write_descrs_fields,
- write_descrs_arrays=self.write_descrs_arrays,
+ write_descrs_fields=self.__write_descrs_fields,
+ write_descrs_arrays=self.__write_descrs_arrays,
)
arraycopydescr1 = FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE,
FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1])
diff --git a/rpython/jit/metainterp/test/test_warmspot.py
b/rpython/jit/metainterp/test/test_warmspot.py
--- a/rpython/jit/metainterp/test/test_warmspot.py
+++ b/rpython/jit/metainterp/test/test_warmspot.py
@@ -624,7 +624,7 @@
pass
def setup_descrs(self):
- pass
+ return []
def get_latest_descr(self, deadframe):
assert isinstance(deadframe, FakeDeadFrame)
diff --git a/rpython/jit/metainterp/warmspot.py
b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -277,6 +277,7 @@
for vinfo in vinfos:
if vinfo is not None:
vinfo.finish()
+ self.metainterp_sd.finish_setup_descrs()
if self.cpu.translate_support_code:
self.annhelper.finish()
diff --git a/rpython/tool/algo/bitstring.py b/rpython/tool/algo/bitstring.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/algo/bitstring.py
@@ -0,0 +1,23 @@
+
+
+def make_bitstring(lst):
+ "NOT_RPYTHON"
+ if not lst:
+ return ''
+ num_bits = max(lst) + 1
+ num_bytes = (num_bits + 7) // 8
+ entries = [0] * num_bytes
+ for x in lst:
+ assert x >= 0
+ entries[x >> 3] |= 1 << (x & 7)
+ return ''.join(map(chr, entries))
+
+def bitcheck(bitstring, n):
+ assert n >= 0
+ byte_number = n >> 3
+ if byte_number >= len(bitstring):
+ return False
+ return (ord(bitstring[byte_number]) & (1 << (n & 7))) != 0
+
+def num_bits(bitstring):
+ return len(bitstring) << 3
diff --git a/rpython/tool/algo/test/test_bitstring.py
b/rpython/tool/algo/test/test_bitstring.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/algo/test/test_bitstring.py
@@ -0,0 +1,25 @@
+from rpython.tool.algo.bitstring import *
+from hypothesis import given, strategies
+
+def test_make():
+ assert make_bitstring([]) == ''
+ assert make_bitstring([0]) == '\x01'
+ assert make_bitstring([7]) == '\x80'
+ assert make_bitstring([8]) == '\x00\x01'
+ assert make_bitstring([2, 4, 20]) == '\x14\x00\x10'
+
+def test_bitcheck():
+ assert bitcheck('\x01', 0) is True
+ assert bitcheck('\x01', 1) is False
+ assert bitcheck('\x01', 10) is False
+ assert [n for n in range(32) if bitcheck('\x14\x00\x10', n)] == [2, 4, 20]
+
+@given(strategies.lists(strategies.integers(min_value=0, max_value=299)))
+def test_random(lst):
+ bitstring = make_bitstring(lst)
+ assert set([n for n in range(300) if bitcheck(bitstring, n)]) == set(lst)
+
+def test_num_bits():
+ assert num_bits('') == 0
+ assert num_bits('a') == 8
+ assert num_bits('bcd') == 24
diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py
b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -1,3 +1,4 @@
+import py
from rpython.rtyper.lltypesystem import lltype
from rpython.translator.translator import TranslationContext, graphof
from rpython.translator.backendopt.writeanalyze import WriteAnalyzer, top_set
@@ -314,6 +315,7 @@
assert T1 == T2
def test_cutoff(self):
+ py.test.skip("cutoff: disabled")
from rpython.rlib.unroll import unrolling_iterable
cutoff = 20
attrs = unrolling_iterable(["s%s" % i for i in range(cutoff + 5)])
diff --git a/rpython/translator/backendopt/writeanalyze.py
b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -4,10 +4,14 @@
top_set = object()
empty_set = frozenset()
-CUTOFF = 3000
+# CUTOFF is disabled, as it gave a strangely not-working-any-more effect
+# if the size of the result grows past that bound. The main user was
+# optimizeopt/heap.py (force_from_effectinfo), which has been rewritten
+# to be happy with any size now.
+#CUTOFF = 3000
class WriteAnalyzer(graphanalyze.GraphAnalyzer):
- cutoff = CUTOFF
+ #cutoff = CUTOFF
def bottom_result(self):
return empty_set
@@ -25,8 +29,8 @@
if other is top_set:
return top_set
result.update(other)
- if len(result) > self.cutoff:
- return top_set
+ #if len(result) > self.cutoff:
+ # return top_set
return result
def finalize_builder(self, result):
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
