Author: Armin Rigo <[email protected]>
Branch:
Changeset: r72312:93a8f5aeb3bc
Date: 2014-07-02 11:18 +0200
http://bitbucket.org/pypy/pypy/changeset/93a8f5aeb3bc/
Log: Issue #1783
Improve array.extend(x) by not requiring an intermediate list in
case there isn't one, e.g. if x is a generator or iterator.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -967,6 +967,13 @@
"""
return self.unpackiterable(w_iterable, expected_length)
+ def listview_no_unpack(self, w_iterable):
+ """ Same as listview() if cheap. If 'w_iterable' is something like
+ a generator, for example, then return None instead.
+ May return None anyway.
+ """
+ return None
+
def listview_bytes(self, w_list):
""" Return a list of unwrapped strings out of a list of strings. If the
argument is not a list or does not contain only strings, return None.
diff --git a/pypy/module/array/interp_array.py
b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -15,6 +15,7 @@
interp2app, interpindirect2app, unwrap_spec)
from pypy.interpreter.typedef import (
GetSetProperty, TypeDef, make_weakref_descr)
+from pypy.interpreter.generator import GeneratorIterator
from pypy.module._file.interp_file import W_File
from pypy.objspace.std.floatobject import W_FloatObject
@@ -630,6 +631,10 @@
def make_array(mytype):
W_ArrayBase = globals()['W_ArrayBase']
+ unpack_driver = jit.JitDriver(name='unpack_array',
+ greens=['tp'],
+ reds=['self', 'w_iterator'])
+
class W_Array(W_ArrayBase):
itemsize = mytype.bytes
typecode = mytype.typecode
@@ -739,31 +744,64 @@
space = self.space
oldlen = self.len
newlen = oldlen
- try:
- # optimized case for arrays of integers or floats
- if mytype.unwrap == 'int_w':
- lst = space.listview_int(w_seq)
- elif mytype.unwrap == 'float_w':
- lst = space.listview_float(w_seq)
- else:
- lst = None
- if lst is not None:
- self.setlen(oldlen + len(lst))
+
+ # optimized case for arrays of integers or floats
+ if mytype.unwrap == 'int_w':
+ lst = space.listview_int(w_seq)
+ elif mytype.unwrap == 'float_w':
+ lst = space.listview_float(w_seq)
+ else:
+ lst = None
+ if lst is not None:
+ self.setlen(oldlen + len(lst))
+ try:
buf = self.buffer
for num in lst:
buf[newlen] = self.item_from_int_or_float(num)
newlen += 1
- return
- #
- # this is the general case
- lst_w = space.listview(w_seq)
+ except OperationError:
+ self.setlen(newlen)
+ raise
+ return
+
+ # this is the common case: w_seq is a list or a tuple
+ lst_w = space.listview_no_unpack(w_seq)
+ if lst_w is not None:
self.setlen(oldlen + len(lst_w))
- for w_num in lst_w:
- self.buffer[newlen] = self.item_w(w_num)
- newlen += 1
- finally:
- if self.len != newlen:
- self.setlen(newlen)
+ buf = self.buffer
+ try:
+ for w_num in lst_w:
+ # note: self.item_w() might invoke arbitrary code.
+ # In case it resizes the same array, then strange
+ # things may happen, but as we don't reload 'buf'
+ # we know that one is big enough for all items
+ # (so at least we avoid crashes)
+ buf[newlen] = self.item_w(w_num)
+ newlen += 1
+ except OperationError:
+ if buf == self.buffer:
+ self.setlen(newlen)
+ raise
+ return
+
+ self._fromiterable(w_seq)
+
+ def _fromiterable(self, w_seq):
+ # a more careful case if w_seq happens to be a very large
+ # iterable: don't copy the items into some intermediate list
+ w_iterator = self.space.iter(w_seq)
+ tp = self.space.type(w_iterator)
+ while True:
+ unpack_driver.jit_merge_point(tp=tp, self=self,
+ w_iterator=w_iterator)
+ space = self.space
+ try:
+ w_item = space.next(w_iterator)
+ except OperationError, e:
+ if not e.match(space, space.w_StopIteration):
+ raise
+ break # done
+ self.descr_append(space, w_item)
def extend(self, w_iterable, accept_different_array=False):
space = self.space
@@ -806,8 +844,9 @@
def descr_append(self, space, w_x):
x = self.item_w(w_x)
- self.setlen(self.len + 1)
- self.buffer[self.len - 1] = x
+ index = self.len
+ self.setlen(index + 1)
+ self.buffer[index] = x
# List interface
def descr_count(self, space, w_val):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -421,14 +421,19 @@
assert expected_length >= 0
return self.fixedview(w_obj, expected_length, unroll=True)
+ def listview_no_unpack(self, w_obj):
+ if type(w_obj) is W_ListObject:
+ return w_obj.getitems()
+ elif isinstance(w_obj, W_AbstractTupleObject) and
self._uses_tuple_iter(w_obj):
+ return w_obj.getitems_copy()
+ elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+ return w_obj.getitems()
+ else:
+ return None
+
def listview(self, w_obj, expected_length=-1):
- if type(w_obj) is W_ListObject:
- t = w_obj.getitems()
- elif isinstance(w_obj, W_AbstractTupleObject) and
self._uses_tuple_iter(w_obj):
- t = w_obj.getitems_copy()
- elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
- t = w_obj.getitems()
- else:
+ t = self.listview_no_unpack(w_obj)
+ if t is None:
return ObjSpace.unpackiterable(self, w_obj, expected_length)
if expected_length != -1 and len(t) != expected_length:
raise self._wrap_expected_length(expected_length, len(t))
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
