Author: Matti Picus <[email protected]>
Branch: cpyext-ext
Changeset: r84888:8939df11b5d2
Date: 2016-06-02 18:33 +0300
http://bitbucket.org/pypy/pypy/changeset/8939df11b5d2/
Log: merge nonmovable-list into branch
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -184,7 +184,7 @@
def can_move(self, addr):
return False
- def malloc_fixedsize_nonmovable(self, typeid):
+ def malloc_fixed_or_varsize_nonmovable(self, typeid, length):
raise MemoryError
def pin(self, addr):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -718,8 +718,9 @@
return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
- def malloc_fixedsize_nonmovable(self, typeid):
- obj = self.external_malloc(typeid, 0, alloc_young=True)
+ def malloc_fixed_or_varsize_nonmovable(self, typeid, length):
+ # length==0 for fixedsize
+ obj = self.external_malloc(typeid, length, alloc_young=True)
return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -628,8 +628,9 @@
return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
- def malloc_fixedsize_nonmovable(self, typeid):
- obj = self.external_malloc(typeid, 0, alloc_young=True)
+ def malloc_fixed_or_varsize_nonmovable(self, typeid, length):
+ # length==0 for fixedsize
+ obj = self.external_malloc(typeid, length, alloc_young=True)
return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
diff --git a/rpython/memory/gctransform/framework.py
b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -557,9 +557,10 @@
getfn(func,
[SomeAddress()],
annmodel.s_None)
- self.malloc_nonmovable_ptr = getfn(GCClass.malloc_fixedsize_nonmovable,
- [s_gc, s_typeid16],
- s_gcref)
+ self.malloc_nonmovable_ptr = getfn(
+ GCClass.malloc_fixed_or_varsize_nonmovable,
+ [s_gc, s_typeid16, annmodel.SomeInteger()],
+ s_gcref)
self.register_finalizer_ptr = getfn(GCClass.register_finalizer,
[s_gc,
@@ -800,12 +801,16 @@
c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
has_light_finalizer)
+ is_varsize = op.opname.endswith('_varsize') or flags.get('varsize')
+
if flags.get('nonmovable'):
- assert op.opname == 'malloc'
- assert not flags.get('varsize')
+ if not is_varsize:
+ v_length = rmodel.inputconst(lltype.Signed, 0)
+ else:
+ v_length = op.args[-1]
malloc_ptr = self.malloc_nonmovable_ptr
- args = [self.c_const_gc, c_type_id]
- elif not op.opname.endswith('_varsize') and not flags.get('varsize'):
+ args = [self.c_const_gc, c_type_id, v_length]
+ elif not is_varsize:
zero = flags.get('zero', False)
if (self.malloc_fast_ptr is not None and
not c_has_finalizer.value and
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1001,3 +1001,259 @@
def specialize_call(self, hop):
hop.exception_cannot_occur()
return hop.genop('gc_gettypeid', hop.args_v, resulttype=lltype.Signed)
+
+# ____________________________________________________________
+
+
+class _rawptr_missing_item(object):
+ pass
+_rawptr_missing_item = _rawptr_missing_item()
+
+
+class _ResizableListSupportingRawPtr(list):
+ """Calling this class is a no-op after translation.
+
+ Before translation, it returns a new instance of
+ _ResizableListSupportingRawPtr, on which
+ rgc.nonmoving_raw_ptr_for_resizable_list() might be
+ used if needed. For now, only supports lists of chars.
+ """
+ __slots__ = ('_raw_items',) # either None or a rffi.CCHARP
+
+ def __init__(self, lst):
+ self._raw_items = None
+ self.__from_list(lst)
+
+ def __resize(self):
+ """Called before an operation changes the size of the list"""
+ if self._raw_items is not None:
+ list.__init__(self, self.__as_list())
+ self._raw_items = None
+
+ def __from_list(self, lst):
+ """Initialize the list from a copy of the list 'lst'."""
+ assert isinstance(lst, list)
+ for x in lst:
+ assert isinstance(x, str) and len(x) == 1
+ if self is lst:
+ return
+ if len(self) != len(lst):
+ self.__resize()
+ if self._raw_items is None:
+ list.__init__(self, lst)
+ else:
+ assert len(self) == self._raw_items._obj.getlength() == len(lst)
+ for i in range(len(self)):
+ self._raw_items[i] = lst[i]
+
+ def __as_list(self):
+ """Return a list (the same or a different one) which contains the
+ items in the regular way."""
+ if self._raw_items is None:
+ return self
+ length = self._raw_items._obj.getlength()
+ assert length == len(self)
+ return [self._raw_items[i] for i in range(length)]
+
+ def __getitem__(self, index):
+ if self._raw_items is None:
+ return list.__getitem__(self, index)
+ if index < 0:
+ index += len(self)
+ if not (0 <= index < len(self)):
+ raise IndexError
+ return self._raw_items[index]
+
+ def __setitem__(self, index, new):
+ if self._raw_items is None:
+ return list.__setitem__(self, index, new)
+ if index < 0:
+ index += len(self)
+ if not (0 <= index < len(self)):
+ raise IndexError
+ self._raw_items[index] = new
+
+ def __delitem__(self, index):
+ self.__resize()
+ list.__delitem__(self, index)
+
+ def __getslice__(self, i, j):
+ return list.__getslice__(self.__as_list(), i, j)
+
+ def __setslice__(self, i, j, new):
+ lst = self.__as_list()
+ list.__setslice__(lst, i, j, new)
+ self.__from_list(lst)
+
+ def __delslice__(self, i, j):
+ lst = self.__as_list()
+ list.__delslice__(lst, i, j)
+ self.__from_list(lst)
+
+ def __iter__(self):
+ try:
+ i = 0
+ while True:
+ yield self[i]
+ i += 1
+ except IndexError:
+ pass
+
+ def __reversed__(self):
+ i = len(self)
+ while i > 0:
+ i -= 1
+ yield self[i]
+
+ def __contains__(self, item):
+ return list.__contains__(self.__as_list(), item)
+
+ def __add__(self, other):
+ if isinstance(other, _ResizableListSupportingRawPtr):
+ other = other.__as_list()
+ return list.__add__(self.__as_list(), other)
+
+ def __radd__(self, other):
+ if isinstance(other, _ResizableListSupportingRawPtr):
+ other = other.__as_list()
+ return list.__add__(other, self.__as_list())
+
+ def __iadd__(self, other):
+ self.__resize()
+ return list.__iadd__(self, other)
+
+ def __eq__(self, other):
+ return list.__eq__(self.__as_list(), other)
+ def __ne__(self, other):
+ return list.__ne__(self.__as_list(), other)
+ def __ge__(self, other):
+ return list.__ge__(self.__as_list(), other)
+ def __gt__(self, other):
+ return list.__gt__(self.__as_list(), other)
+ def __le__(self, other):
+ return list.__le__(self.__as_list(), other)
+ def __lt__(self, other):
+ return list.__lt__(self.__as_list(), other)
+
+ def __mul__(self, other):
+ return list.__mul__(self.__as_list(), other)
+
+ def __rmul__(self, other):
+ return list.__mul__(self.__as_list(), other)
+
+ def __imul__(self, other):
+ self.__resize()
+ return list.__imul__(self, other)
+
+ def __repr__(self):
+ return '_ResizableListSupportingRawPtr(%s)' % (
+ list.__repr__(self.__as_list()),)
+
+ def append(self, object):
+ self.__resize()
+ return list.append(self, object)
+
+ def count(self, value):
+ return list.count(self.__as_list(), value)
+
+ def extend(self, iterable):
+ self.__resize()
+ return list.extend(self, iterable)
+
+ def index(self, value, *start_stop):
+ return list.index(self.__as_list(), value, *start_stop)
+
+ def insert(self, index, object):
+ self.__resize()
+ return list.insert(self, index, object)
+
+ def pop(self, *opt_index):
+ self.__resize()
+ return list.pop(self, *opt_index)
+
+ def remove(self, value):
+ self.__resize()
+ return list.remove(self, value)
+
+ def reverse(self):
+ lst = self.__as_list()
+ list.reverse(lst)
+ self.__from_list(lst)
+
+ def sort(self, *args, **kwds):
+ lst = self.__as_list()
+ list.sort(lst, *args, **kwds)
+ self.__from_list(lst)
+
+ def _nonmoving_raw_ptr_for_resizable_list(self):
+ if self._raw_items is None:
+ existing_items = list(self)
+ from rpython.rtyper.lltypesystem import lltype, rffi
+ self._raw_items = lltype.malloc(rffi.CCHARP.TO, len(self),
+ flavor='raw', immortal=True)
+ self.__from_list(existing_items)
+ assert self._raw_items is not None
+ return self._raw_items
+
+def resizable_list_supporting_raw_ptr(lst):
+ return _ResizableListSupportingRawPtr(lst)
+
+def nonmoving_raw_ptr_for_resizable_list(lst):
+ assert isinstance(lst, _ResizableListSupportingRawPtr)
+ return lst._nonmoving_raw_ptr_for_resizable_list()
+
+
+def _check_resizable_list_of_chars(s_list):
+ from rpython.annotator import model as annmodel
+ from rpython.rlib import debug
+ if annmodel.s_None.contains(s_list):
+ return # "None", will likely be generalized later
+ if not isinstance(s_list, annmodel.SomeList):
+ raise Exception("not a list, got %r" % (s_list,))
+ if not isinstance(s_list.listdef.listitem.s_value,
+ (annmodel.SomeChar, annmodel.SomeImpossibleValue)):
+ raise debug.NotAListOfChars
+ s_list.listdef.resize() # must be resizable
+
+class Entry(ExtRegistryEntry):
+ _about_ = resizable_list_supporting_raw_ptr
+
+ def compute_result_annotation(self, s_list):
+ _check_resizable_list_of_chars(s_list)
+ return s_list
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ return hop.inputarg(hop.args_r[0], 0)
+
+class Entry(ExtRegistryEntry):
+ _about_ = nonmoving_raw_ptr_for_resizable_list
+
+ def compute_result_annotation(self, s_list):
+ from rpython.rtyper.lltypesystem import lltype, rffi
+ from rpython.rtyper.llannotation import SomePtr
+ _check_resizable_list_of_chars(s_list)
+ return SomePtr(rffi.CCHARP)
+
+ def specialize_call(self, hop):
+ v_list = hop.inputarg(hop.args_r[0], 0)
+ hop.exception_cannot_occur() # ignoring MemoryError
+ return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list,
+ v_list)
+
+def ll_nonmovable_raw_ptr_for_resizable_list(ll_list):
+ from rpython.rtyper.lltypesystem import lltype, rffi
+ array = ll_list.items
+ if can_move(array):
+ length = ll_list.length
+ new_array = lltype.malloc(lltype.typeOf(ll_list).TO.items.TO, length,
+ nonmovable=True)
+ i = 0
+ while i < length:
+ new_array[i] = array[i]
+ i += 1
+ ll_list.items = new_array
+ array = new_array
+ ptr = lltype.direct_arrayitems(array)
+ # ptr is a Ptr(FixedSizeArray(Char, 1)). Cast it to a rffi.CCHARP
+ return rffi.cast(rffi.CCHARP, ptr)
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -1,6 +1,6 @@
from rpython.rtyper.test.test_llinterp import gengraph, interpret
from rpython.rtyper.error import TyperError
-from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
from rpython.rlib import rgc # Force registration of gc.collect
import gc
import py, sys
@@ -254,6 +254,153 @@
assert typer.custom_trace_funcs == [(TP, trace_func)]
+def test_nonmoving_raw_ptr_for_resizable_list():
+ def f(n):
+ lst = ['a', 'b', 'c']
+ lst = rgc.resizable_list_supporting_raw_ptr(lst)
+ lst.append(chr(n))
+ assert lst[3] == chr(n)
+ assert lst[-1] == chr(n)
+ #
+ ptr = rgc.nonmoving_raw_ptr_for_resizable_list(lst)
+ assert lst[:] == ['a', 'b', 'c', chr(n)]
+ assert lltype.typeOf(ptr) == rffi.CCHARP
+ assert [ptr[i] for i in range(4)] == ['a', 'b', 'c', chr(n)]
+ #
+ lst[-3] = 'X'
+ assert ptr[1] == 'X'
+ ptr[2] = 'Y'
+ assert lst[-2] == 'Y'
+ #
+ addr = rffi.cast(lltype.Signed, ptr)
+ ptr = rffi.cast(rffi.CCHARP, addr)
+ rgc.collect() # should not move lst.items
+ lst[-4] = 'g'
+ assert ptr[0] == 'g'
+ ptr[3] = 'H'
+ assert lst[-1] == 'H'
+ return lst
+ #
+ # direct untranslated run
+ lst = f(35)
+ assert isinstance(lst, rgc._ResizableListSupportingRawPtr)
+ #
+ # llinterp run
+ interpret(f, [35])
+ #
+ # compilation with the GC transformer
+ import subprocess
+ from rpython.translator.interactive import Translation
+ #
+ def main(argv):
+ f(len(argv))
+ print "OK!"
+ return 0
+ #
+ t = Translation(main, gc="incminimark")
+ t.disable(['backendopt'])
+ t.set_backend_extra_options(c_debug_defines=True)
+ exename = t.compile()
+ data = subprocess.check_output([str(exename), '.', '.', '.'])
+ assert data.strip().endswith('OK!')
+
+def test_ListSupportingRawPtr_direct():
+ lst = ['a', 'b', 'c']
+ lst = rgc.resizable_list_supporting_raw_ptr(lst)
+
+ def check_nonresizing():
+ assert lst[1] == lst[-2] == 'b'
+ lst[1] = 'X'
+ assert lst[1] == 'X'
+ lst[-1] = 'Y'
+ assert lst[1:3] == ['X', 'Y']
+ assert lst[-2:9] == ['X', 'Y']
+ lst[1:2] = 'B'
+ assert lst[:] == ['a', 'B', 'Y']
+ assert list(iter(lst)) == ['a', 'B', 'Y']
+ assert list(reversed(lst)) == ['Y', 'B', 'a']
+ assert 'B' in lst
+ assert 'b' not in lst
+ assert p[0] == 'a'
+ assert p[1] == 'B'
+ assert p[2] == 'Y'
+ assert lst + ['*'] == ['a', 'B', 'Y', '*']
+ assert ['*'] + lst == ['*', 'a', 'B', 'Y']
+ assert lst + lst == ['a', 'B', 'Y', 'a', 'B', 'Y']
+ base = ['8']
+ base += lst
+ assert base == ['8', 'a', 'B', 'Y']
+ assert lst == ['a', 'B', 'Y']
+ assert ['a', 'B', 'Y'] == lst
+ assert ['a', 'B', 'Z'] != lst
+ assert ['a', 'B', 'Z'] > lst
+ assert ['a', 'B', 'Z'] >= lst
+ assert lst * 2 == ['a', 'B', 'Y', 'a', 'B', 'Y']
+ assert 2 * lst == ['a', 'B', 'Y', 'a', 'B', 'Y']
+ assert lst.count('B') == 1
+ assert lst.index('Y') == 2
+ lst.reverse()
+ assert lst == ['Y', 'B', 'a']
+ lst.sort()
+ assert lst == ['B', 'Y', 'a']
+ lst.sort(reverse=True)
+ assert lst == ['a', 'Y', 'B']
+ lst[1] = 'b'
+ lst[2] = 'c'
+ assert list(lst) == ['a', 'b', 'c']
+
+ p = lst
+ check_nonresizing()
+ assert lst._raw_items is None
+ lst._nonmoving_raw_ptr_for_resizable_list()
+ p = lst._raw_items
+ check_nonresizing()
+ assert lst._raw_items == p
+ assert p[0] == 'a'
+ assert p[1] == 'b'
+ assert p[2] == 'c'
+
+ def do_resizing_operation():
+ del lst[1]
+ yield ['a', 'c']
+
+ lst[:2] = ['X']
+ yield ['X', 'c']
+
+ del lst[:2]
+ yield ['c']
+
+ x = lst
+ x += ['t']
+ yield ['a', 'b', 'c', 't']
+
+ x = lst
+ x *= 3
+ yield ['a', 'b', 'c'] * 3
+
+ lst.append('f')
+ yield ['a', 'b', 'c', 'f']
+
+ lst.extend('fg')
+ yield ['a', 'b', 'c', 'f', 'g']
+
+ lst.insert(1, 'k')
+ yield ['a', 'k', 'b', 'c']
+
+ n = lst.pop(1)
+ assert n == 'b'
+ yield ['a', 'c']
+
+ lst.remove('c')
+ yield ['a', 'b']
+
+ assert lst == ['a', 'b', 'c']
+ for expect in do_resizing_operation():
+ assert lst == expect
+ assert lst._raw_items is None
+ lst = ['a', 'b', 'c']
+ lst = rgc.resizable_list_supporting_raw_ptr(lst)
+ lst._nonmoving_raw_ptr_for_resizable_list()
# ____________________________________________________________
@@ -368,7 +515,6 @@
assert fq._triggered == 1
def test_finalizer_trigger_calls_too_much(self):
- from rpython.rtyper.lltypesystem import lltype, rffi
external_func = rffi.llexternal("foo", [], lltype.Void)
# ^^^ with release_gil=True
class X(object):
diff --git a/rpython/rtyper/lltypesystem/lltype.py
b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -2174,7 +2174,8 @@
def malloc(T, n=None, flavor='gc', immortal=False, zero=False,
- track_allocation=True, add_memory_pressure=False):
+ track_allocation=True, add_memory_pressure=False,
+ nonmovable=False):
assert flavor in ('gc', 'raw')
if zero or immortal:
initialization = 'example'
@@ -2200,7 +2201,8 @@
@analyzer_for(malloc)
def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None,
- s_track_allocation=None, s_add_memory_pressure=None):
+ s_track_allocation=None, s_add_memory_pressure=None,
+ s_nonmovable=None):
assert (s_n is None or s_n.knowntype == int
or issubclass(s_n.knowntype, base_int))
assert s_T.is_constant()
@@ -2218,6 +2220,7 @@
assert s_track_allocation is None or s_track_allocation.is_constant()
assert (s_add_memory_pressure is None or
s_add_memory_pressure.is_constant())
+ assert s_nonmovable is None or s_nonmovable.is_constant()
# not sure how to call malloc() for the example 'p' in the
# presence of s_extraargs
r = SomePtr(Ptr(s_T.const))
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py
b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -659,6 +659,7 @@
a[3] = 30
a[4] = 40
b0 = direct_arrayitems(a)
+ assert typeOf(b0) == Ptr(FixedSizeArray(Signed, 1))
b1 = direct_ptradd(b0, 1)
b2 = direct_ptradd(b1, 1)
b3 = direct_ptradd(b0, 3)
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -348,7 +348,7 @@
@typer_for(lltype.malloc)
def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None,
- i_add_memory_pressure=None):
+ i_add_memory_pressure=None, i_nonmovable=None):
assert hop.args_s[0].is_constant()
vlist = [hop.inputarg(lltype.Void, arg=0)]
opname = 'malloc'
@@ -357,8 +357,10 @@
(i_flavor, lltype.Void),
(i_zero, None),
(i_track_allocation, None),
- (i_add_memory_pressure, None))
- (v_flavor, v_zero, v_track_allocation, v_add_memory_pressure) = kwds_v
+ (i_add_memory_pressure, None),
+ (i_nonmovable, None))
+ (v_flavor, v_zero, v_track_allocation,
+ v_add_memory_pressure, v_nonmovable) = kwds_v
flags = {'flavor': 'gc'}
if v_flavor is not None:
flags['flavor'] = v_flavor.value
@@ -368,6 +370,8 @@
flags['track_allocation'] = v_track_allocation.value
if i_add_memory_pressure is not None:
flags['add_memory_pressure'] = v_add_memory_pressure.value
+ if i_nonmovable is not None:
+ flags['nonmovable'] = v_nonmovable
vlist.append(hop.inputconst(lltype.Void, flags))
assert 1 <= hop.nb_args <= 2
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
