Author: Antonio Cuni <[email protected]>
Branch:
Changeset: r91396:d3895494491b
Date: 2017-05-24 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/d3895494491b/
Log: merge the faster-rstruct-2 branch: this makes struct.pack and
struct.pack_into much faster by using raw_store or gc_store_indexed
whenever possible. Moreover, it enables the existing struct.unpack
fast path to all the existing buffer types, whereas previously it
was enabled only for strings
diff too long, truncating to 2000 out of 3450 lines
diff --git a/pypy/module/__pypy__/bytebuffer.py
b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -2,29 +2,8 @@
# A convenient read-write buffer. Located here for want of a better place.
#
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import ByteBuffer
from pypy.interpreter.gateway import unwrap_spec
-from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
-
-
-class ByteBuffer(Buffer):
- _immutable_ = True
-
- def __init__(self, len):
- self.data = ['\x00'] * len
- self.readonly = False
-
- def getlength(self):
- return len(self.data)
-
- def getitem(self, index):
- return self.data[index]
-
- def setitem(self, index, char):
- self.data[index] = char
-
- def get_raw_address(self):
- return nonmoving_raw_ptr_for_resizable_list(self.data)
@unwrap_spec(length=int)
def bytebuffer(space, length):
diff --git a/pypy/module/_cffi_backend/cbuffer.py
b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -6,13 +6,13 @@
from pypy.objspace.std.bufferobject import W_Buffer
from pypy.interpreter.buffer import SimpleView
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
from rpython.rtyper.annlowlevel import llstr
from rpython.rtyper.lltypesystem import rffi
from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-class LLBuffer(Buffer):
+class LLBuffer(RawBuffer):
_immutable_ = True
def __init__(self, raw_cdata, size):
@@ -35,7 +35,7 @@
def getslice(self, start, stop, step, size):
if step == 1:
return rffi.charpsize2str(rffi.ptradd(self.raw_cdata, start), size)
- return Buffer.getslice(self, start, stop, step, size)
+ return RawBuffer.getslice(self, start, stop, step, size)
def setslice(self, start, string):
raw_cdata = rffi.ptradd(self.raw_cdata, start)
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,11 +1,10 @@
from rpython.rtyper.lltypesystem import rffi
-
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
# XXX not the most efficient implementation
-class RawFFIBuffer(Buffer):
+class RawFFIBuffer(RawBuffer):
_immutable_ = True
def __init__(self, datainstance):
diff --git a/pypy/module/array/interp_array.py
b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,5 +1,5 @@
from rpython.rlib import jit, rgc
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rlib.rarithmetic import ovfcheck, widen
from rpython.rlib.unroll import unrolling_iterable
@@ -796,7 +796,7 @@
v.typecode = k
unroll_typecodes = unrolling_iterable(types.keys())
-class ArrayBuffer(Buffer):
+class ArrayBuffer(RawBuffer):
_immutable_ = True
def __init__(self, array, readonly):
@@ -840,7 +840,7 @@
return rffi.charpsize2str(rffi.ptradd(data, start), size)
finally:
self.array._charbuf_stop()
- return Buffer.getslice(self, start, stop, step, size)
+ return RawBuffer.getslice(self, start, stop, step, size)
def get_raw_address(self):
return self.array._charbuf_start()
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -552,6 +552,7 @@
to compress, call the flush() method to finish the compression process,
and return what is left in the internal buffers."""
+ assert data is not None
datasize = len(data)
if datasize == 0:
@@ -662,6 +663,7 @@
was found after the end of stream, it'll be ignored and saved in
unused_data attribute."""
+ assert data is not None
if not self.running:
raise oefmt(self.space.w_EOFError,
"end of stream was already found")
@@ -715,6 +717,7 @@
use an instance of BZ2Compressor instead. The compresslevel parameter, if
given, must be a number between 1 and 9."""
+ assert data is not None
if compresslevel < 1 or compresslevel > 9:
raise oefmt(space.w_ValueError,
"compresslevel must be between 1 and 9")
@@ -757,6 +760,7 @@
Decompress data in one shot. If you want to decompress data sequentially,
use an instance of BZ2Decompressor instead."""
+ assert data is not None
in_bufsize = len(data)
if in_bufsize == 0:
return space.newbytes("")
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -22,8 +22,7 @@
from pypy.interpreter.buffer import BufferView
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.argument import Arguments
-
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
from rpython.rlib.unroll import unrolling_iterable
from rpython.rlib.objectmodel import specialize, not_rpython
from rpython.tool.sourcetools import func_renamer
@@ -427,7 +426,7 @@
fq = FQ()
-class CBuffer(Buffer):
+class CBuffer(RawBuffer):
_immutable_ = True
def __init__(self, view):
self.view = view
diff --git a/pypy/module/micronumpy/concrete.py
b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -3,7 +3,7 @@
from rpython.rlib import jit, rgc
from rpython.rlib.rarithmetic import ovfcheck
from rpython.rlib.listsort import make_timsort_class
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
from rpython.rlib.debug import make_sure_not_resized
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
@@ -702,7 +702,8 @@
def __del__(self):
free_raw_storage(self.storage)
-class ArrayData(Buffer):
+
+class ArrayData(RawBuffer):
_immutable_ = True
def __init__(self, impl, readonly):
self.impl = impl
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -3,7 +3,7 @@
from pypy.interpreter.typedef import TypeDef
from pypy.interpreter.gateway import interp2app, unwrap_spec
from rpython.rlib import rmmap, rarithmetic, objectmodel
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
from rpython.rlib.rstring import StringBuilder
@@ -330,7 +330,7 @@
return OperationError(space.w_SystemError, space.newtext('%s' % e))
-class MMapBuffer(Buffer):
+class MMapBuffer(RawBuffer):
_immutable_ = True
def __init__(self, space, mmap, readonly):
@@ -350,7 +350,7 @@
if step == 1:
return self.mmap.getslice(start, size)
else:
- return Buffer.getslice(self, start, stop, step, size)
+ return RawBuffer.getslice(self, start, stop, step, size)
def setitem(self, index, char):
self.check_valid_writeable()
diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -34,29 +34,13 @@
i = 0
while i < n:
i += 1
- struct.unpack('<i', a) # ID: unpack
+ struct.unpack('i', a) # ID: unpack
return i
log = self.run(main, [1000])
assert log.result == 1000
loop, = log.loops_by_filename(self.filepath)
assert loop.match_by_id('unpack', """
guard_not_invalidated(descr=...)
- p90 = newstr(4)
- call_n(ConstClass(copy_raw_to_string), i55, p90, 0, 4,
descr=<Callv 0 irii EF=5>)
- guard_no_exception(descr=...)
- i91 = strgetitem(p90, 0)
- i92 = strgetitem(p90, 1)
- i93 = int_lshift(i92, 8)
- i94 = int_or(i91, i93)
- i95 = strgetitem(p90, 2)
- i96 = int_lshift(i95, 16)
- i97 = int_or(i94, i96)
- i98 = strgetitem(p90, 3)
- i99 = int_ge(i98, 128)
- guard_false(i99, descr=...)
- i100 = int_lshift(i98, 24)
- i101 = int_or(i97, i100)
- i102 = getfield_raw_i(#, descr=<FieldS pypysig_long_struct.c_value
0>)
- i103 = int_lt(i102, 0)
- guard_false(i103, descr=...)
+ i66 = raw_load_i(i53, 0, descr=<ArrayS 4>)
+ --TICK--
""")
diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py
b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -30,32 +30,33 @@
loop, = log.loops_by_filename(self.filepath)
# This could, of course stand some improvement, to remove all these
# arithmatic ops, but we've removed all the core overhead.
- assert loop.match_by_id("pack", """
- guard_not_invalidated(descr=...)
- # struct.pack
- %s
- i11 = int_and(i4, 255)
- i13 = int_rshift(i4, 8)
- i14 = int_and(i13, 255)
- i16 = int_rshift(i13, 8)
- i17 = int_and(i16, 255)
- i19 = int_rshift(i16, 8)
- i20 = int_and(i19, 255)
- """ % extra)
+ if sys.byteorder == 'little':
+ # on little endian machines, we take the fast path and store the
+ # value using gc_store_indexed
+ assert loop.match_by_id("pack", """
+ guard_not_invalidated(descr=...)
+ # struct.pack
+ %s
+ p75 = newstr(4)
+ gc_store_indexed(p75, 0, _, 1, _, 4, descr=...)
+ """ % extra)
+ else:
+ assert loop.match_by_id("pack", """
+ guard_not_invalidated(descr=...)
+ # struct.pack
+ %s
+ i11 = int_and(i4, 255)
+ i13 = int_rshift(i4, 8)
+ i14 = int_and(i13, 255)
+ i16 = int_rshift(i13, 8)
+ i17 = int_and(i16, 255)
+ i19 = int_rshift(i16, 8)
+ i20 = int_and(i19, 255)
+ """ % extra)
if sys.byteorder == 'little':
- # the newstr and the strsetitems are because the string is forced,
- # which is in turn because the optimizer doesn't know how to
handle a
- # gc_load_indexed_i on a virtual string. It could be improved, but
it
- # is also true that in real life cases struct.unpack is called on
- # strings which come from the outside, so it's a minor issue.
assert loop.match_by_id("unpack", """
# struct.unpack
- p88 = newstr(4)
- strsetitem(p88, 0, i11)
- strsetitem(p88, 1, i14)
- strsetitem(p88, 2, i17)
- strsetitem(p88, 3, i20)
i91 = gc_load_indexed_i(p88, 0, 1, _, -4)
""")
else:
@@ -93,27 +94,106 @@
assert loop.match_by_id('pack', """
guard_not_invalidated(descr=...)
# struct.pack
+ p85 = newstr(8)
+ gc_store_indexed(p85, 0, -1, 1, _, 4, descr=...)
%s
- i11 = int_and(i4, 255)
- i13 = int_rshift(i4, 8)
- i14 = int_and(i13, 255)
- i16 = int_rshift(i13, 8)
- i17 = int_and(i16, 255)
- i19 = int_rshift(i16, 8)
- i20 = int_and(i19, 255)
+ gc_store_indexed(p85, 4, _, 1, _, 4, descr=...)
""" % extra)
assert loop.match_by_id('unpack', """
# struct.unpack
- p88 = newstr(8)
- strsetitem(p88, 0, 255)
- strsetitem(p88, 1, 255)
- strsetitem(p88, 2, 255)
- strsetitem(p88, 3, 255)
- strsetitem(p88, 4, i11)
- strsetitem(p88, 5, i14)
- strsetitem(p88, 6, i17)
- strsetitem(p88, 7, i20)
i90 = gc_load_indexed_i(p88, 0, 1, _, -4)
i91 = gc_load_indexed_i(p88, 4, 1, _, -4)
""")
+
+ def test_unpack_raw_buffer(self):
+ def main(n):
+ import array
+ import struct
+ buf = struct.pack('H', 0x1234)
+ buf = array.array('b', buf)
+ i = 1
+ res = 0
+ while i < n:
+ val = struct.unpack("h", buf)[0] # ID: unpack
+ res += val
+ i += 1
+ return res
+ log = self.run(main, [1000])
+ assert log.result == main(1000)
+ loop, = log.loops_by_filename(self.filepath)
+ assert loop.match_by_id('unpack', """
+ guard_not_invalidated(descr=...)
+ i65 = raw_load_i(i49, 0, descr=<ArrayS 2>)
+ """)
+
+ def test_unpack_bytearray(self):
+ def main(n):
+ import struct
+ buf = struct.pack('H', 0x1234)
+ buf = bytearray(buf)
+ i = 1
+ res = 0
+ while i < n:
+ val = struct.unpack("h", buf)[0] # ID: unpack
+ res += val
+ i += 1
+ return res
+ log = self.run(main, [1000])
+ assert log.result == main(1000)
+ loop, = log.loops_by_filename(self.filepath)
+ # the offset of gc_load_indexed_i used to be the constant 0. However,
+ # now it is 'i46' because we need to add 0 to
+ # W_BytearrayObject._offset
+ assert loop.match_by_id('unpack', """
+ guard_not_invalidated(descr=...)
+ i70 = gc_load_indexed_i(p48, i46, 1, _, -2)
+ """)
+
+ def test_pack_into_raw_buffer(self):
+ def main(n):
+ import array
+ import struct
+ buf = array.array('b', '\x00'*8)
+ i = 1
+ while i < n:
+ struct.pack_into("h", buf, 4, i) # ID: pack_into
+ i += 1
+ return i
+ log = self.run(main, [1000])
+ assert log.result == main(1000)
+ loop, = log.loops_by_filename(self.filepath)
+ assert loop.match_by_id('pack_into', """\
+ guard_not_invalidated(descr=...)
+ i65 = int_le(i58, 32767)
+ guard_true(i65, descr=...)
+ raw_store(i55, 4, i58, descr=<ArrayS 2>)
+ """)
+
+ def test_pack_into_bytearray(self):
+ def main(n):
+ import struct
+ buf = bytearray(8)
+ i = 1
+ while i < n:
+ struct.pack_into("h", buf, 4, i) # ID: pack_into
+ i += 1
+ return i
+ log = self.run(main, [1000])
+ assert log.result == main(1000)
+ loop, = log.loops_by_filename(self.filepath)
+ assert loop.match_by_id('pack_into', """\
+ guard_not_invalidated(descr=...)
+ p68 = getfield_gc_r(p14, descr=<FieldP
pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__data \d+>)
+ i69 = getfield_gc_i(p68, descr=<FieldS list.length \d+>)
+ i70 = getfield_gc_i(p14, descr=<FieldS
pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__offset \d+>)
+ i71 = int_sub(i69, i70)
+ i73 = int_sub(i71, 4)
+ i75 = int_lt(i73, 2)
+ guard_false(i75, descr=...)
+ i77 = int_le(i62, 32767)
+ guard_true(i77, descr=...)
+ p78 = getfield_gc_r(p68, descr=<FieldP list.items \d+>)
+ i81 = int_add(4, i70)
+ gc_store_indexed(p78, i81, i62, 1, _, 2, descr=<ArrayS 2>)
+ """)
diff --git a/pypy/module/struct/formatiterator.py
b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -2,7 +2,6 @@
maxint, intmask)
from rpython.rlib import jit
from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rstruct.error import StructError
from rpython.rlib.rstruct.formatiterator import FormatIterator
@@ -10,11 +9,15 @@
class PackFormatIterator(FormatIterator):
- def __init__(self, space, args_w, size):
+ def __init__(self, space, wbuf, args_w):
self.space = space
self.args_w = args_w
self.args_index = 0
- self.result = StringBuilder(size)
+ self.pos = 0
+ self.wbuf = wbuf
+
+ def advance(self, count):
+ self.pos += count
# This *should* be always unroll safe, the only way to get here is by
# unroll the interpret function, which means the fmt is const, and thus
@@ -31,8 +34,10 @@
@jit.unroll_safe
def align(self, mask):
- pad = (-self.result.getlength()) & mask
- self.result.append_multiple_char('\x00', pad)
+ pad = (-self.pos) & mask
+ for i in range(self.pos, self.pos+pad):
+ self.wbuf.setitem(i, '\x00')
+ self.advance(pad)
def finished(self):
if self.args_index != len(self.args_w):
@@ -140,13 +145,20 @@
if self.pos != self.length:
raise StructError("unpack str size too long for format")
+ def can_advance(self, count):
+ end = self.pos + count
+ return end <= self.length
+
+ def advance(self, count):
+ if not self.can_advance(count):
+ raise StructError("unpack str size too short for format")
+ self.pos += count
+
def read(self, count):
- end = self.pos + count
- if end > self.length:
- raise StructError("unpack str size too short for format")
- s = self.buf.getslice(self.pos, end, 1, count)
- self.pos = end
- return s
+ curpos = self.pos
+ end = curpos + count
+ self.advance(count) # raise if we are out of bound
+ return self.buf.getslice(curpos, end, 1, count)
@specialize.argtype(1)
def appendobj(self, value):
@@ -182,9 +194,8 @@
def get_pos(self):
return self.pos
- def get_buffer_as_string_maybe(self):
- string, pos = self.buf.as_str_and_offset_maybe()
- return string, pos+self.pos
+ def get_buffer_and_pos(self):
+ return self.buf, self.pos
def skip(self, size):
self.read(size) # XXX, could avoid taking the slice
diff --git a/pypy/module/struct/interp_struct.py
b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -1,5 +1,6 @@
from rpython.rlib import jit
from rpython.rlib.buffer import SubBuffer
+from rpython.rlib.mutbuffer import MutableStringBuffer
from rpython.rlib.rstruct.error import StructError, StructOverflowError
from rpython.rlib.rstruct.formatiterator import CalcSizeFormatIterator
@@ -40,18 +41,17 @@
def _pack(space, format, args_w):
"""Return string containing values v1, v2, ... packed according to fmt."""
- if jit.isconstant(format):
- size = _calcsize(space, format)
- else:
- size = 8
- fmtiter = PackFormatIterator(space, args_w, size)
+ size = _calcsize(space, format)
+ wbuf = MutableStringBuffer(size)
+ fmtiter = PackFormatIterator(space, wbuf, args_w)
try:
fmtiter.interpret(format)
except StructOverflowError as e:
raise OperationError(space.w_OverflowError, space.newtext(e.msg))
except StructError as e:
raise OperationError(get_error(space), space.newtext(e.msg))
- return fmtiter.result.build()
+ assert fmtiter.pos == wbuf.getlength(), 'missing .advance() or wrong
calcsize()'
+ return wbuf.finish()
@unwrap_spec(format='text')
@@ -59,22 +59,28 @@
return space.newbytes(_pack(space, format, args_w))
-# XXX inefficient
@unwrap_spec(format='text', offset=int)
def pack_into(space, format, w_buffer, offset, args_w):
""" Pack the values v1, v2, ... according to fmt.
Write the packed bytes into the writable buffer buf starting at offset
"""
- res = _pack(space, format, args_w)
+ size = _calcsize(space, format)
buf = space.getarg_w('w*', w_buffer)
if offset < 0:
offset += buf.getlength()
- size = len(res)
if offset < 0 or (buf.getlength() - offset) < size:
raise oefmt(get_error(space),
"pack_into requires a buffer of at least %d bytes",
size)
- buf.setslice(offset, res)
+ #
+ wbuf = SubBuffer(buf, offset, size)
+ fmtiter = PackFormatIterator(space, wbuf, args_w)
+ try:
+ fmtiter.interpret(format)
+ except StructOverflowError as e:
+ raise OperationError(space.w_OverflowError, space.newtext(e.msg))
+ except StructError as e:
+ raise OperationError(get_error(space), space.newtext(e.msg))
def _unpack(space, format, buf):
diff --git a/pypy/module/struct/test/test_struct.py
b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -491,7 +491,7 @@
class AppTestFastPath(object):
- spaceconfig = dict(usemodules=['struct', '__pypy__'])
+ spaceconfig = dict(usemodules=['array', 'struct', '__pypy__'])
def setup_class(cls):
from rpython.rlib.rstruct import standardfmttable
@@ -510,7 +510,43 @@
from rpython.rlib.rstruct import standardfmttable
standardfmttable.ALLOW_SLOWPATH = True
+ def test_unpack_simple(self):
+ buf = self.struct.pack("iii", 0, 42, 43)
+ assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
def test_unpack_from(self):
buf = self.struct.pack("iii", 0, 42, 43)
offset = self.struct.calcsize("i")
assert self.struct.unpack_from("ii", buf, offset) == (42, 43)
+
+ def test_unpack_bytearray(self):
+ data = self.struct.pack("iii", 0, 42, 43)
+ buf = bytearray(data)
+ assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
+ def test_unpack_array(self):
+ import array
+ data = self.struct.pack("iii", 0, 42, 43)
+ buf = array.array('c', data)
+ assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
+ def test_pack_into_bytearray(self):
+ expected = self.struct.pack("ii", 42, 43)
+ buf = bytearray(len(expected))
+ self.struct.pack_into("ii", buf, 0, 42, 43)
+ assert buf == expected
+
+ def test_pack_into_bytearray_padding(self):
+ expected = self.struct.pack("xxi", 42)
+ buf = bytearray(len(expected))
+ self.struct.pack_into("xxi", buf, 0, 42)
+ assert buf == expected
+
+ def test_pack_into_bytearray_delete(self):
+ expected = self.struct.pack("i", 42)
+ # force W_BytearrayObject._delete_from_start
+ buf = bytearray(64)
+ del buf[:8]
+ self.struct.pack_into("i", buf, 0, 42)
+ buf = buf[:len(expected)]
+ assert buf == expected
diff --git a/pypy/objspace/std/bytearrayobject.py
b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -6,10 +6,11 @@
from rpython.rlib.debug import check_list_of_chars, check_nonneg
from rpython.rtyper.lltypesystem import rffi
from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
- nonmoving_raw_ptr_for_resizable_list)
+ nonmoving_raw_ptr_for_resizable_list)
from rpython.rlib import jit
-from rpython.rlib.buffer import Buffer
-
+from rpython.rlib.buffer import (GCBuffer,
+ get_gc_data_for_list_of_chars,
+ get_gc_data_offset_for_list_of_chars)
from pypy.interpreter.baseobjspace import W_Root
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
@@ -1257,7 +1258,8 @@
start += step
-class BytearrayBuffer(Buffer):
[email protected]
+class BytearrayBuffer(GCBuffer):
_immutable_ = True
def __init__(self, ba, readonly=False):
@@ -1287,7 +1289,7 @@
if start != 0 or stop != len(data):
data = data[start:stop]
return "".join(data)
- return Buffer.getslice(self, start, stop, step, size)
+ return GCBuffer.getslice(self, start, stop, step, size)
def setslice(self, start, string):
# No bounds checks.
@@ -1302,6 +1304,16 @@
p = rffi.ptradd(p, ba._offset)
return p
+ @staticmethod
+ def _get_gc_data_offset():
+ return get_gc_data_offset_for_list_of_chars()
+
+ def _get_gc_data_extra_offset(self):
+ return self.ba._offset
+
+ def _get_gc_data(self):
+ return get_gc_data_for_list_of_chars(self.ba._data)
+
@specialize.argtype(1)
def _memcmp(selfvalue, buffer, length):
diff --git a/pypy/objspace/std/marshal_impl.py
b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -1,5 +1,5 @@
from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.mutbuffer import MutableStringBuffer
from rpython.rlib.rstruct import ieee
from rpython.rlib.unroll import unrolling_iterable
@@ -190,9 +190,9 @@
def pack_float(f):
- result = StringBuilder(8)
- ieee.pack_float(result, f, 8, False)
- return result.build()
+ buf = MutableStringBuffer(8)
+ ieee.pack_float(buf, 0, f, 8, False)
+ return buf.finish()
def unpack_float(s):
return ieee.unpack_float(s, False)
diff --git a/pypy/objspace/std/unicodeobject.py
b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -4,6 +4,7 @@
compute_hash, compute_unique_id, import_from_mixin,
enforceargs)
from rpython.rlib.buffer import StringBuffer
+from rpython.rlib.mutbuffer import MutableStringBuffer
from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
from rpython.rlib.runicode import (
make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
@@ -84,10 +85,12 @@
def readbuf_w(self, space):
from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
- builder = StringBuilder(len(self._value) * UNICODE_SIZE)
+ buf = MutableStringBuffer(len(self._value) * UNICODE_SIZE)
+ pos = 0
for unich in self._value:
- pack_unichar(unich, builder)
- return StringBuffer(builder.build())
+ pack_unichar(unich, buf, pos)
+ pos += UNICODE_SIZE
+ return StringBuffer(buf.finish())
def writebuf_w(self, space):
raise oefmt(space.w_TypeError,
diff --git a/rpython/jit/backend/arm/test/test_llop.py
b/rpython/jit/backend/arm/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/arm/test/test_llop.py
@@ -0,0 +1,9 @@
+from rpython.jit.backend.arm.test.support import JitARMMixin
+from rpython.jit.metainterp.test.test_llop import TestLLOp as _TestLLOp
+
+
+class TestLLOp(JitARMMixin, _TestLLOp):
+ # for the individual tests see
+ # ====> ../../../metainterp/test/test_llop.py
+ pass
+
diff --git a/rpython/jit/backend/llgraph/runner.py
b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -716,16 +716,28 @@
else:
return self.bh_raw_load_i(struct, offset, descr)
+ def _get_int_type_from_size(self, size):
+ if size == 1:
+ return rffi.UCHAR
+ elif size == 2:
+ return rffi.USHORT
+ elif size == 4:
+ return rffi.UINT
+ elif size == 8:
+ return rffi.ULONGLONG
+ elif size == -1:
+ return rffi.SIGNEDCHAR
+ elif size == -2:
+ return rffi.SHORT
+ elif size == -4:
+ return rffi.INT
+ elif size == -8:
+ return rffi.LONGLONG
+ else:
+ raise NotImplementedError(size)
+
def bh_gc_load_indexed_i(self, struct, index, scale, base_ofs, bytes):
- if bytes == 1: T = rffi.UCHAR
- elif bytes == 2: T = rffi.USHORT
- elif bytes == 4: T = rffi.UINT
- elif bytes == 8: T = rffi.ULONGLONG
- elif bytes == -1: T = rffi.SIGNEDCHAR
- elif bytes == -2: T = rffi.SHORT
- elif bytes == -4: T = rffi.INT
- elif bytes == -8: T = rffi.LONGLONG
- else: raise NotImplementedError(bytes)
+ T = self._get_int_type_from_size(bytes)
x = llop.gc_load_indexed(T, struct, index, scale, base_ofs)
return lltype.cast_primitive(lltype.Signed, x)
@@ -735,6 +747,30 @@
return llop.gc_load_indexed(longlong.FLOATSTORAGE,
struct, index, scale, base_ofs)
+ def bh_gc_store_indexed_i(self, struct, index, val, scale, base_ofs, bytes,
+ descr):
+ T = self._get_int_type_from_size(bytes)
+ val = lltype.cast_primitive(T, val)
+ if descr.A.OF == lltype.SingleFloat:
+ val = longlong.int2singlefloat(val)
+ llop.gc_store_indexed(lltype.Void, struct, index, val, scale, base_ofs)
+
+ def bh_gc_store_indexed_f(self, struct, index, val, scale, base_ofs, bytes,
+ descr):
+ if bytes != 8:
+ raise Exception("gc_store_indexed_f is only for 'double'!")
+ val = longlong.getrealfloat(val)
+ llop.gc_store_indexed(lltype.Void, struct, index, val, scale, base_ofs)
+
+ def bh_gc_store_indexed(self, struct, index, val, scale, base_ofs, bytes,
+ descr):
+ if descr.A.OF == lltype.Float:
+ self.bh_gc_store_indexed_f(struct, index, val, scale, base_ofs,
+ bytes, descr)
+ else:
+ self.bh_gc_store_indexed_i(struct, index, val, scale, base_ofs,
+ bytes, descr)
+
def bh_increment_debug_counter(self, addr):
p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr)
p[0] += 1
diff --git a/rpython/jit/backend/llsupport/llmodel.py
b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -496,6 +496,7 @@
@specialize.argtype(1)
def write_float_at_mem(self, gcref, ofs, newvalue):
llop.raw_store(lltype.Void, gcref, ofs, newvalue)
+ write_float_at_mem._annenforceargs_ = [None, None, None,
longlong.r_float_storage]
# ____________________________________________________________
@@ -754,6 +755,16 @@
offset = base_ofs + scale * index
return self.read_float_at_mem(addr, offset)
+ def bh_gc_store_indexed_i(self, addr, index, val, scale, base_ofs, bytes,
+ descr):
+ offset = base_ofs + scale * index
+ self.write_int_at_mem(addr, offset, bytes, val)
+
+ def bh_gc_store_indexed_f(self, addr, index, val, scale, base_ofs, bytes,
+ descr):
+ offset = base_ofs + scale * index
+ self.write_float_at_mem(addr, offset, val)
+
def bh_new(self, sizedescr):
return self.gc_ll_descr.gc_malloc(sizedescr)
diff --git a/rpython/jit/backend/x86/test/test_llop.py
b/rpython/jit/backend/x86/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_llop.py
@@ -0,0 +1,14 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test.test_llop import TestLLOp as _TestLLOp
+
+
+class TestLLOp(Jit386Mixin, _TestLLOp):
+ # for the individual tests see
+ # ====> ../../../metainterp/test/test_llop.py
+
+ # do NOT test the blackhole implementation of gc_store_indexed. It cannot
+ # work inside tests because llmodel.py:bh_gc_store_indexed_* receive a
+ # symbolic as the offset. It is not a problem because it is tested anyway
+ # by the same test in test_metainterp.py
+ TEST_BLACKHOLE = False
+
diff --git a/rpython/jit/backend/x86/test/test_strstorage.py
b/rpython/jit/backend/x86/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/test/test_strstorage.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
-from rpython.jit.metainterp.test.test_strstorage import TestStrStorage as
_TestStrStorage
-
-
-class TestStrStorage(Jit386Mixin, _TestStrStorage):
- # for the individual tests see
- # ====> ../../../metainterp/test/test_strstorage.py
- pass
diff --git a/rpython/jit/codewriter/jtransform.py
b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1131,6 +1131,26 @@
[op.args[0], op.args[1],
op.args[2], op.args[3], c_bytes], op.result)
+ def rewrite_op_gc_store_indexed(self, op):
+ T = op.args[2].concretetype
+ kind = getkind(T)[0]
+ assert kind != 'r'
+ descr = self.cpu.arraydescrof(rffi.CArray(T))
+ if (not isinstance(op.args[3], Constant) or
+ not isinstance(op.args[4], Constant)):
+ raise NotImplementedError("gc_store_indexed: 'scale' and
'base_ofs'"
+ " should be constants")
+ # According to the comment in resoperation.py, "itemsize is not signed
+ # (always > 0)", so we don't need the "bytes = -bytes" line which is
+ # in rewrite_op_gc_load_indexed
+ bytes = descr.get_item_size_in_bytes()
+ c_bytes = Constant(bytes, lltype.Signed)
+ return SpaceOperation('gc_store_indexed_%s' % kind,
+ [op.args[0], op.args[1], op.args[2],
+ op.args[3], op.args[4], c_bytes, descr], None)
+
+
+
def _rewrite_equality(self, op, opname):
arg0, arg1 = op.args
if isinstance(arg0, Constant) and not arg0.value:
diff --git a/rpython/jit/metainterp/blackhole.py
b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1478,6 +1478,18 @@
def bhimpl_gc_load_indexed_f(cpu, addr, index, scale, base_ofs, bytes):
return cpu.bh_gc_load_indexed_f(addr, index,scale,base_ofs, bytes)
+ @arguments("cpu", "r", "i", "i", "i", "i", "i", "d")
+ def bhimpl_gc_store_indexed_i(cpu, addr, index, val, scale, base_ofs,
bytes,
+ arraydescr):
+ return cpu.bh_gc_store_indexed_i(addr, index, val, scale,base_ofs,
bytes,
+ arraydescr)
+
+ @arguments("cpu", "r", "i", "f", "i", "i", "i", "d")
+ def bhimpl_gc_store_indexed_f(cpu, addr, index, val, scale, base_ofs,
bytes,
+ arraydescr):
+ return cpu.bh_gc_store_indexed_f(addr, index, val, scale,base_ofs,
bytes,
+ arraydescr)
+
@arguments("r", "d", "d")
def bhimpl_record_quasiimmut_field(struct, fielddescr, mutatefielddescr):
pass
diff --git a/rpython/jit/metainterp/executor.py
b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -251,6 +251,25 @@
else:
return BoxInt(cpu.bh_raw_load_i(addr, offset, arraydescr))
+def do_gc_store_indexed(cpu, _, addrbox, indexbox, valuebox, scalebox,
+ base_ofsbox, bytesbox, arraydescr):
+ addr = addrbox.getref_base()
+ index = indexbox.getint()
+ scale = scalebox.getint()
+ base_ofs = base_ofsbox.getint()
+ bytes = bytesbox.getint()
+ if arraydescr.is_array_of_pointers():
+ raise AssertionError("cannot store GC pointers in gc_store_indexed for
now")
+ elif arraydescr.is_array_of_floats():
+ floatval = valuebox.getfloatstorage()
+ cpu.bh_gc_store_indexed_f(addr, index, floatval, scale, base_ofs,
bytes,
+ arraydescr)
+ else:
+ intval = valuebox.getint()
+ cpu.bh_gc_store_indexed_i(addr, index, intval, scale, base_ofs, bytes,
+ arraydescr)
+
+
def exec_new_with_vtable(cpu, descr):
return cpu.bh_new_with_vtable(descr)
diff --git a/rpython/jit/metainterp/pyjitpl.py
b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -827,6 +827,21 @@
self._remove_symbolics(scalebox),
self._remove_symbolics(baseofsbox), bytesbox)
+ @arguments("box", "box", "box", "box", "box", "box", "descr")
+ def _opimpl_gc_store_indexed(self, addrbox, indexbox, valuebox,
+ scalebox, baseofsbox, bytesbox,
+ arraydescr):
+ return self.execute_with_descr(rop.GC_STORE_INDEXED,
+ arraydescr,
+ addrbox,
+ indexbox,
+ valuebox,
+ self._remove_symbolics(scalebox),
+ self._remove_symbolics(baseofsbox),
+ bytesbox)
+ opimpl_gc_store_indexed_i = _opimpl_gc_store_indexed
+ opimpl_gc_store_indexed_f = _opimpl_gc_store_indexed
+
@arguments("box")
def opimpl_hint_force_virtualizable(self, box):
self.metainterp.gen_store_back_in_vable(box)
diff --git a/rpython/jit/metainterp/test/support.py
b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -19,7 +19,8 @@
supports_floats=True,
supports_longlong=False,
supports_singlefloats=False,
- translationoptions={}, **kwds):
+ translationoptions={},
+ backendopt_inline_threshold=0, **kwds):
from rpython.jit.codewriter import support
class FakeJitCell(object):
@@ -59,7 +60,7 @@
FakeWarmRunnerState.enable_opts = {}
func._jit_unroll_safe_ = True
- rtyper = support.annotate(func, values,
+ rtyper = support.annotate(func, values, inline=backendopt_inline_threshold,
translationoptions=translationoptions)
graphs = rtyper.annotator.translator.graphs
testself.all_graphs = graphs
diff --git a/rpython/jit/metainterp/test/test_llop.py
b/rpython/jit/metainterp/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -0,0 +1,72 @@
+import py
+import sys
+import struct
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.test.test_llop import (BaseLLOpTest, str_gc_load,
+ newlist_and_gc_store)
+from rpython.jit.codewriter import longlong
+from rpython.jit.metainterp.history import getkind
+from rpython.jit.metainterp.test.support import LLJitMixin
+
+
+class TestLLOp(BaseLLOpTest, LLJitMixin):
+
+ # for the individual tests see
+ # ====> ../../../rtyper/test/test_llop.py
+ TEST_BLACKHOLE = True
+
+ def gc_load_from_string(self, TYPE, buf, offset):
+ def f(offset):
+ return str_gc_load(TYPE, buf, offset)
+ res = self.interp_operations(f, [offset], supports_singlefloats=True)
+ #
+ kind = getkind(TYPE)[0] # 'i' or 'f'
+ self.check_operations_history({'gc_load_indexed_%s' % kind: 1,
+ 'finish': 1})
+ #
+ if TYPE == lltype.SingleFloat:
+ # interp_operations returns the int version of r_singlefloat, but
+ # our tests expects to receive an r_singlefloat: let's convert it
+ # back!
+ return longlong.int2singlefloat(res)
+ return res
+
+ def newlist_and_gc_store(self, TYPE, value, expected):
+ def f(value):
+ lst = newlist_and_gc_store(TYPE, value)
+ got = ''.join(lst)
+ if got != expected:
+ # I'm not sure why, but if I use an assert, the test doesn't
fail
+ raise ValueError('got != expected')
+ return len(got)
+ #
+ if self.TEST_BLACKHOLE:
+ # we pass a big inline_threshold to ensure that
+ # newlist_and_gc_store is inlined, else the blackhole does not see
+ # (and thus we do not test!) the llop.gc_store_indexed
+ threshold = 33
+ else:
+ threshold = 0
+ return self.interp_operations(f, [value], supports_singlefloats=True,
+ backendopt_inline_threshold=threshold)
+
+
+ def test_force_virtual_str_storage(self):
+ byteorder = sys.byteorder
+ size = rffi.sizeof(lltype.Signed)
+ def f(val):
+ if byteorder == 'little':
+ x = chr(val) + '\x00'*(size-1)
+ else:
+ x = '\x00'*(size-1) + chr(val)
+ return str_gc_load(lltype.Signed, x, 0)
+ res = self.interp_operations(f, [42], supports_singlefloats=True)
+ assert res == 42
+ self.check_operations_history({
+ 'newstr': 1, # str forcing
+ 'strsetitem': 1, # str forcing
+ 'call_pure_r': 1, # str forcing (copystrcontent)
+ 'guard_no_exception': 1, # str forcing
+ 'gc_load_indexed_i': 1, # str_storage_getitem
+ 'finish': 1
+ })
diff --git a/rpython/jit/metainterp/test/test_strstorage.py
b/rpython/jit/metainterp/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/jit/metainterp/test/test_strstorage.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import py
-import sys
-import struct
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.strstorage import str_storage_getitem
-from rpython.rlib.test.test_strstorage import BaseStrStorageTest
-from rpython.jit.codewriter import longlong
-from rpython.jit.metainterp.history import getkind
-from rpython.jit.metainterp.test.support import LLJitMixin
-
-class TestStrStorage(BaseStrStorageTest, LLJitMixin):
-
- # for the individual tests see
- # ====> ../../../rlib/test/test_strstorage.py
-
- def str_storage_getitem(self, TYPE, buf, offset):
- def f():
- return str_storage_getitem(TYPE, buf, offset)
- res = self.interp_operations(f, [], supports_singlefloats=True)
- #
- kind = getkind(TYPE)[0] # 'i' or 'f'
- self.check_operations_history({'gc_load_indexed_%s' % kind: 1,
- 'finish': 1})
- #
- if TYPE == lltype.SingleFloat:
- # interp_operations returns the int version of r_singlefloat, but
- # our tests expects to receive an r_singlefloat: let's convert it
- # back!
- return longlong.int2singlefloat(res)
- return res
-
- #def str_storage_supported(self, TYPE):
- # py.test.skip('this is not a JIT test')
-
- def test_force_virtual_str_storage(self):
- byteorder = sys.byteorder
- size = rffi.sizeof(lltype.Signed)
- def f(val):
- if byteorder == 'little':
- x = chr(val) + '\x00'*(size-1)
- else:
- x = '\x00'*(size-1) + chr(val)
- return str_storage_getitem(lltype.Signed, x, 0)
- res = self.interp_operations(f, [42], supports_singlefloats=True)
- assert res == 42
- self.check_operations_history({
- 'newstr': 1, # str forcing
- 'strsetitem': 1, # str forcing
- 'call_pure_r': 1, # str forcing (copystrcontent)
- 'guard_no_exception': 1, # str forcing
- 'gc_load_indexed_i': 1, # str_storage_getitem
- 'finish': 1
- })
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -1,13 +1,55 @@
"""
Buffer protocol support.
"""
-from rpython.rlib.rgc import (
- nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.lltypesystem.rlist import LIST_OF
+from rpython.rtyper.annlowlevel import llstr
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+ nonmoving_raw_ptr_for_resizable_list,
+ ll_for_resizable_list)
from rpython.rlib.signature import signature
from rpython.rlib import types
+from rpython.rlib import rawstorage
+
+ALLOW_UNALIGNED_ACCESS = rawstorage.misaligned_is_fine
+
[email protected]()
+def is_alignment_correct(TYPE, index):
+ if ALLOW_UNALIGNED_ACCESS:
+ return True
+ try:
+ rawstorage._check_alignment(TYPE, index)
+ except rawstorage.AlignmentError:
+ return False
+ else:
+ return True
+
+
+class CannotRead(Exception):
+ """
+ Exception raised by Buffer.typed_read in case it is not possible to
+ accomplish the request. This might be because it is not supported by the
+ specific type of buffer, or because of alignment issues.
+ """
+
+class CannotWrite(Exception):
+ """
+ Raised by Buffer.typed_write in case it is not possible to accomplish the
+ request
+ """
class Buffer(object):
- """Base class for buffers of bytes"""
+ """
+ Base class for buffers of bytes.
+
+ Most probably, you do NOT want to use this as a lone base class, but
+ either inherit from RawBuffer or GCBuffer, so that you automatically get
+ the proper implementation of typed_read and typed_write.
+ """
_attrs_ = ['readonly']
_immutable_ = True
@@ -25,14 +67,6 @@
# May be overridden.
return self.getslice(0, self.getlength(), 1, self.getlength())
- def as_str_and_offset_maybe(self):
- """
- If the buffer is backed by a string, return a pair (string, offset),
- where offset is the offset inside the string where the buffer start.
- Else, return (None, 0).
- """
- return None, 0
-
def getitem(self, index):
"Returns the index'th character in the buffer."
raise NotImplementedError # Must be overriden. No bounds checks.
@@ -60,7 +94,141 @@
for i in range(len(string)):
self.setitem(start + i, string[i])
-class ByteBuffer(Buffer):
+ @jit.look_inside_iff(lambda self, index, count:
+ jit.isconstant(count) and count <= 8)
+ def setzeros(self, index, count):
+ for i in range(index, index+count):
+ self.setitem(i, '\x00')
+
+ @specialize.ll_and_arg(1)
+ def typed_read(self, TP, byte_offset):
+ """
+ Read the value of type TP starting at byte_offset. No bounds checks
+ """
+ raise CannotRead
+
+ @specialize.ll_and_arg(1)
+ def typed_write(self, TP, byte_offset, value):
+ """
+ Write the value of type TP at byte_offset. No bounds checks
+ """
+ raise CannotWrite
+
+
+class RawBuffer(Buffer):
+ """
+ A buffer which is baked by a raw, non-movable memory area. It implementes
+ typed_read and typed_write in terms of get_raw_address(), llop.raw_load,
+ llop.raw_store.
+
+ NOTE: this assumes that get_raw_address() is cheap. Do not use this as a
+ base class if get_raw_address() is potentially costly, like for example if
+ you call rgc.nonmoving_raw_ptr_for_resizable_list
+ """
+ _immutable_ = True
+
+ @specialize.ll_and_arg(1)
+ def typed_read(self, TP, byte_offset):
+ """
+ Read the value of type TP starting at byte_offset. No bounds checks
+ """
+ if not is_alignment_correct(TP, byte_offset):
+ raise CannotRead
+ ptr = self.get_raw_address()
+ return llop.raw_load(TP, ptr, byte_offset)
+
+ @specialize.ll_and_arg(1)
+ def typed_write(self, TP, byte_offset, value):
+ """
+ Write the value of type TP at byte_offset. No bounds checks
+ """
+ if self.readonly or not is_alignment_correct(TP, byte_offset):
+ raise CannotWrite
+ ptr = self.get_raw_address()
+ value = lltype.cast_primitive(TP, value)
+ return llop.raw_store(lltype.Void, ptr, byte_offset, value)
+
+
+class GCBuffer(Buffer):
+ """
+ Base class for a buffer which is baked by a GC-managed memory area. You
+ MUST also decorate the class with @GCBuffer.decorate: it implements
+ typed_read and typed_write in terms of llop.gc_load_indexed and
+ llop.gc_store_indexed.
+ """
+ _attrs_ = ['readonly', 'value']
+ _immutable_ = True
+
+ @staticmethod
+ def decorate(targetcls):
+ """
+ Create and attach specialized versions of typed_{read,write}. We need
to
+ do this becase the JIT codewriters mandates that base_ofs is an
+ RPython constant.
+ """
+ if targetcls.__bases__ != (GCBuffer,):
+ raise ValueError("@GCBuffer.decorate should be used only on "
+ "GCBuffer subclasses")
+
+ base_ofs = targetcls._get_gc_data_offset()
+ scale_factor = llmemory.sizeof(lltype.Char)
+
+ @specialize.ll_and_arg(1)
+ def typed_read(self, TP, byte_offset):
+ if not is_alignment_correct(TP, byte_offset):
+ raise CannotRead
+ lldata = self._get_gc_data()
+ byte_offset += self._get_gc_data_extra_offset()
+ return llop.gc_load_indexed(TP, lldata, byte_offset,
+ scale_factor, base_ofs)
+
+ @specialize.ll_and_arg(1)
+ def typed_write(self, TP, byte_offset, value):
+ if self.readonly or not is_alignment_correct(TP, byte_offset):
+ raise CannotWrite
+ lldata = self._get_gc_data()
+ byte_offset += self._get_gc_data_extra_offset()
+ value = lltype.cast_primitive(TP, value)
+ return llop.gc_store_indexed(lltype.Void, lldata, byte_offset,
value,
+ scale_factor, base_ofs)
+
+ targetcls.typed_read = typed_read
+ targetcls.typed_write = typed_write
+ return targetcls
+
+ @staticmethod
+ def _get_gc_data_offset(self):
+ raise NotImplementedError
+
+ def _get_gc_data_extra_offset(self):
+ return 0
+
+ def _get_gc_data(self):
+ raise NotImplementedError
+
+ @specialize.ll_and_arg(1)
+ def typed_read(self, TP, byte_offset):
+ raise NotImplementedError("You MUST decorate this class with "
+ "@GCBuffer.decorate")
+
+ @specialize.ll_and_arg(1)
+ def typed_write(self, TP, byte_offset, value):
+ raise NotImplementedError("You MUST decorate this class with "
+ "@GCBuffer.decorate")
+
+
+def get_gc_data_for_list_of_chars(data):
+ ll_data = ll_for_resizable_list(data)
+ ll_items = ll_data.items
+ return lltype.cast_opaque_ptr(llmemory.GCREF, ll_items)
+
+def get_gc_data_offset_for_list_of_chars():
+ LIST = LIST_OF(lltype.Char)
+ return llmemory.itemoffsetof(LIST.items.TO, 0)
+
+
[email protected]
+class ByteBuffer(GCBuffer):
_immutable_ = True
def __init__(self, n):
@@ -79,11 +247,21 @@
def get_raw_address(self):
return nonmoving_raw_ptr_for_resizable_list(self.data)
-class StringBuffer(Buffer):
+ def _get_gc_data(self):
+ return get_gc_data_for_list_of_chars(self.data)
+
+ @staticmethod
+ def _get_gc_data_offset():
+ return get_gc_data_offset_for_list_of_chars()
+
+
[email protected]
+class StringBuffer(GCBuffer):
_attrs_ = ['readonly', 'value']
_immutable_ = True
def __init__(self, value):
+ assert value is not None
self.value = value
self.readonly = 1
@@ -93,9 +271,6 @@
def as_str(self):
return self.value
- def as_str_and_offset_maybe(self):
- return self.value, 0
-
def getitem(self, index):
return self.value[index]
@@ -114,6 +289,16 @@
# may still raise ValueError on some GCs
return rffi.get_raw_address_of_string(self.value)
+ @staticmethod
+ def _get_gc_data_offset():
+ return (llmemory.offsetof(STR, 'chars') +
+ llmemory.itemoffsetof(STR.chars, 0))
+
+ def _get_gc_data(self):
+ lls = llstr(self.value)
+ return lltype.cast_opaque_ptr(llmemory.GCREF, lls)
+
+
class SubBuffer(Buffer):
_attrs_ = ['buffer', 'offset', 'size', 'readonly']
_immutable_ = True
@@ -147,12 +332,6 @@
else:
return 0
- def as_str_and_offset_maybe(self):
- string, offset = self.buffer.as_str_and_offset_maybe()
- if string is not None:
- return string, offset + self.offset
- return None, 0
-
def getitem(self, index):
return self.buffer.getitem(self.offset + index)
@@ -176,3 +355,11 @@
from rpython.rtyper.lltypesystem import rffi
ptr = self.buffer.get_raw_address()
return rffi.ptradd(ptr, self.offset)
+
+ @specialize.ll_and_arg(1)
+ def typed_read(self, TP, byte_offset):
+ return self.buffer.typed_read(TP, byte_offset + self.offset)
+
+ @specialize.ll_and_arg(1)
+ def typed_write(self, TP, byte_offset, value):
+ return self.buffer.typed_write(TP, byte_offset + self.offset, value)
diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/mutbuffer.py
@@ -0,0 +1,57 @@
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR, mallocstr
+from rpython.rtyper.annlowlevel import llstr, hlstr
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.buffer import GCBuffer
+from rpython.rlib import jit
+
[email protected]
+class MutableStringBuffer(GCBuffer):
+ """
+ A writeable buffer to incrementally fill a string of a fixed size.
+
+ You can fill the string by calling setitem, setslice and typed_write, and
+ get the result by calling finish().
+
+ After you call finish(), you can no longer modify the buffer. There is no
+ check, you will probably get a segfault after translation.
+
+ You can call finish() only once.
+ """
+ _attrs_ = ['readonly', 'll_val', 'size']
+ _immutable_ = True
+
+ def __init__(self, size):
+ self.readonly = False
+ self.size = size
+ self.ll_val = mallocstr(size)
+
+ def getlength(self):
+ return self.size
+
+ def finish(self):
+ if not self.ll_val:
+ raise ValueError("Cannot call finish() twice")
+ result = hlstr(self.ll_val)
+ self.ll_val = lltype.nullptr(STR)
+ self.readonly = True
+ return result
+
+ def as_str(self):
+ raise ValueError('as_str() is not supported. Use finish() instead')
+
+ def _hlstr(self):
+ assert not we_are_translated() # debug only
+ return hlstr(self.ll_val)
+
+ def setitem(self, index, char):
+ self.ll_val.chars[index] = char
+
+ @staticmethod
+ def _get_gc_data_offset():
+ return (llmemory.offsetof(STR, 'chars') +
+ llmemory.itemoffsetof(STR.chars, 0))
+
+ def _get_gc_data(self):
+ return lltype.cast_opaque_ptr(llmemory.GCREF, self.ll_val)
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1051,17 +1051,17 @@
rgc.nonmoving_raw_ptr_for_resizable_list() might be
used if needed. For now, only supports lists of chars.
"""
- __slots__ = ('_raw_items',) # either None or a rffi.CCHARP
+ __slots__ = ('_ll_list',) # either None or a struct of TYPE=LIST_OF(Char)
def __init__(self, lst):
- self._raw_items = None
+ self._ll_list = None
self.__from_list(lst)
def __resize(self):
"""Called before an operation changes the size of the list"""
- if self._raw_items is not None:
+ if self._ll_list is not None:
list.__init__(self, self.__as_list())
- self._raw_items = None
+ self._ll_list = None
def __from_list(self, lst):
"""Initialize the list from a copy of the list 'lst'."""
@@ -1072,46 +1072,46 @@
return
if len(self) != len(lst):
self.__resize()
- if self._raw_items is None:
+ if self._ll_list is None:
list.__init__(self, lst)
else:
- assert len(self) == self._raw_items._obj.getlength() == len(lst)
+ assert len(self) == self._ll_list.length == len(lst)
for i in range(len(self)):
- self._raw_items[i] = lst[i]
+ self._ll_list.items[i] = lst[i]
def __as_list(self):
"""Return a list (the same or a different one) which contains the
items in the regular way."""
- if self._raw_items is None:
+ if self._ll_list is None:
return self
- length = self._raw_items._obj.getlength()
+ length = self._ll_list.length
assert length == len(self)
- return [self._raw_items[i] for i in range(length)]
+ return [self._ll_list.items[i] for i in range(length)]
def __getitem__(self, index):
- if self._raw_items is None:
+ if self._ll_list is None:
return list.__getitem__(self, index)
if index < 0:
index += len(self)
if not (0 <= index < len(self)):
raise IndexError
- return self._raw_items[index]
+ return self._ll_list.items[index]
def __setitem__(self, index, new):
- if self._raw_items is None:
+ if self._ll_list is None:
return list.__setitem__(self, index, new)
if index < 0:
index += len(self)
if not (0 <= index < len(self)):
raise IndexError
- self._raw_items[index] = new
+ self._ll_list.items[index] = new
def __delitem__(self, index):
self.__resize()
list.__delitem__(self, index)
def __getslice__(self, i, j):
- return list.__getslice__(self.__as_list(), i, j)
+ return self.__class__(list.__getslice__(self.__as_list(), i, j))
def __setslice__(self, i, j, new):
lst = self.__as_list()
@@ -1218,15 +1218,23 @@
list.sort(lst, *args, **kwds)
self.__from_list(lst)
+ def _get_ll_list(self):
+ from rpython.rtyper.lltypesystem import rffi
+ from rpython.rtyper.lltypesystem.rlist import LIST_OF
+ if self._ll_list is None:
+ LIST = LIST_OF(lltype.Char)
+ existing_items = list(self)
+ n = len(self)
+ self._ll_list = lltype.malloc(LIST, immortal=True)
+ self._ll_list.length = n
+ self._ll_list.items = lltype.malloc(LIST.items.TO, n)
+ self.__from_list(existing_items)
+ assert self._ll_list is not None
+ return self._ll_list
+
def _nonmoving_raw_ptr_for_resizable_list(self):
- if self._raw_items is None:
- existing_items = list(self)
- from rpython.rtyper.lltypesystem import lltype, rffi
- self._raw_items = lltype.malloc(rffi.CCHARP.TO, len(self),
- flavor='raw', immortal=True)
- self.__from_list(existing_items)
- assert self._raw_items is not None
- return self._raw_items
+ ll_list = self._get_ll_list()
+ return ll_nonmovable_raw_ptr_for_resizable_list(ll_list)
def resizable_list_supporting_raw_ptr(lst):
return _ResizableListSupportingRawPtr(lst)
@@ -1235,6 +1243,18 @@
assert isinstance(lst, _ResizableListSupportingRawPtr)
return lst._nonmoving_raw_ptr_for_resizable_list()
+def ll_for_resizable_list(lst):
+ """
+ This is the equivalent of llstr(), but for lists. It can be called only if
+ the list has been created by calling resizable_list_supporting_raw_ptr().
+
+ In theory, all the operations on lst are immediately visible also on
+ ll_list. However, support for that is incomplete in
+ _ResizableListSupportingRawPtr and as such, the pointer becomes invalid as
+ soon as you call a resizing operation on lst.
+ """
+ assert isinstance(lst, _ResizableListSupportingRawPtr)
+ return lst._get_ll_list()
def _check_resizable_list_of_chars(s_list):
from rpython.annotator import model as annmodel
@@ -1256,6 +1276,10 @@
return s_list
def specialize_call(self, hop):
+ from rpython.rtyper.lltypesystem.rlist import LIST_OF
+ if hop.args_r[0].LIST != LIST_OF(lltype.Char):
+ raise ValueError('Resizable list of chars does not have the '
+ 'expected low-level type')
hop.exception_cannot_occur()
return hop.inputarg(hop.args_r[0], 0)
@@ -1274,6 +1298,24 @@
return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list,
v_list)
+class Entry(ExtRegistryEntry):
+ _about_ = ll_for_resizable_list
+
+ def compute_result_annotation(self, s_list):
+ from rpython.rtyper.lltypesystem.rlist import LIST_OF
+ from rpython.rtyper.llannotation import lltype_to_annotation
+ _check_resizable_list_of_chars(s_list)
+ LIST = LIST_OF(lltype.Char)
+ return lltype_to_annotation(lltype.Ptr(LIST))
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ assert hop.args_r[0].lowleveltype == hop.r_result.lowleveltype
+ v_ll_list, = hop.inputargs(*hop.args_r)
+ return hop.genop('same_as', [v_ll_list],
+ resulttype = hop.r_result.lowleveltype)
+
+
@jit.dont_look_inside
def ll_nonmovable_raw_ptr_for_resizable_list(ll_list):
"""
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -264,15 +264,23 @@
sign = r_ulonglong(sign)
return (mant, (sign << BITS - MANT_DIG - 1) | exp)
+
+def pack_float(wbuf, pos, x, size, be):
+ unsigned = float_pack(x, size)
+ value = rarithmetic.longlongmask(unsigned)
+ pack_float_to_buffer(wbuf, pos, value, size, be)
+
@jit.unroll_safe
-def pack_float(result, x, size, be):
- l = []
- unsigned = float_pack(x, size)
- for i in range(size):
- l.append(chr((unsigned >> (i * 8)) & 0xFF))
+def pack_float_to_buffer(wbuf, pos, value, size, be):
if be:
- l.reverse()
- result.append("".join(l))
+ # write in reversed order
+ for i in range(size):
+ c = chr((value >> (i * 8)) & 0xFF)
+ wbuf.setitem(pos + size - i - 1, c)
+ else:
+ for i in range(size):
+ c = chr((value >> (i * 8)) & 0xFF)
+ wbuf.setitem(pos+i, c)
@jit.unroll_safe
def pack_float80(result, x, size, be):
diff --git a/rpython/rlib/rstruct/nativefmttable.py
b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -6,10 +6,11 @@
from rpython.rlib import jit, longlong2float
from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_singlefloat, widen
+from rpython.rlib.rarithmetic import r_singlefloat, widen, intmask
from rpython.rlib.rstruct import standardfmttable as std
from rpython.rlib.rstruct.standardfmttable import native_is_bigendian
from rpython.rlib.rstruct.error import StructError
+from rpython.rlib.rstruct.ieee import pack_float_to_buffer
from rpython.rlib.unroll import unrolling_iterable
from rpython.rtyper.lltypesystem import lltype, rffi
from rpython.rtyper.tool import rffi_platform
@@ -26,35 +27,26 @@
# ____________________________________________________________
-range_8_unroll = unrolling_iterable(list(reversed(range(8))))
-range_4_unroll = unrolling_iterable(list(reversed(range(4))))
-
def pack_double(fmtiter):
doubleval = fmtiter.accept_float_arg()
+ if std.pack_fastpath(rffi.DOUBLE)(fmtiter, doubleval):
+ return
+ # slow path
value = longlong2float.float2longlong(doubleval)
- if fmtiter.bigendian:
- for i in range_8_unroll:
- x = (value >> (8*i)) & 0xff
- fmtiter.result.append(chr(x))
- else:
- for i in range_8_unroll:
- fmtiter.result.append(chr(value & 0xff))
- value >>= 8
-
+ pack_float_to_buffer(fmtiter.wbuf, fmtiter.pos, value, 8,
fmtiter.bigendian)
+ fmtiter.advance(8)
def pack_float(fmtiter):
doubleval = fmtiter.accept_float_arg()
floatval = r_singlefloat(doubleval)
+ if std.pack_fastpath(rffi.FLOAT)(fmtiter, floatval):
+ return
+ # slow path
value = longlong2float.singlefloat2uint(floatval)
value = widen(value)
- if fmtiter.bigendian:
- for i in range_4_unroll:
- x = (value >> (8*i)) & 0xff
- fmtiter.result.append(chr(x))
- else:
- for i in range_4_unroll:
- fmtiter.result.append(chr(value & 0xff))
- value >>= 8
+ value = intmask(value)
+ pack_float_to_buffer(fmtiter.wbuf, fmtiter.pos, value, 4,
fmtiter.bigendian)
+ fmtiter.advance(4)
# ____________________________________________________________
#
@@ -151,7 +143,8 @@
if len(unistr) != 1:
raise StructError("expected a unicode string of length 1")
c = unistr[0] # string->char conversion for the annotator
- unichar.pack_unichar(c, fmtiter.result)
+ unichar.pack_unichar(c, fmtiter.wbuf, fmtiter.pos)
+ fmtiter.advance(unichar.UNICODE_SIZE)
@specialize.argtype(0)
def unpack_unichar(fmtiter):
diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -8,19 +8,28 @@
from rpython.rlib.rstruct.formatiterator import FormatIterator
from rpython.rlib.rstruct.error import StructError
from rpython.rlib.objectmodel import specialize
+from rpython.rlib.buffer import StringBuffer
class MasterReader(object):
def __init__(self, s):
- self.input = s
+ self.inputbuf = StringBuffer(s)
+ self.length = len(s)
self.inputpos = 0
+ def can_advance(self, count):
+ end = self.inputpos + count
+ return end <= self.length
+
+ def advance(self, count):
+ if not self.can_advance(count):
+ raise StructError("unpack str size too short for format")
+ self.inputpos += count
+
def read(self, count):
- end = self.inputpos + count
- if end > len(self.input):
- raise StructError("unpack str size too short for format")
- s = self.input[self.inputpos : end]
- self.inputpos = end
- return s
+ curpos = self.inputpos
+ end = curpos + count
+ self.advance(count) # raise if we are out of bound
+ return self.inputbuf.getslice(curpos, end, 1, count)
def align(self, mask):
self.inputpos = (self.inputpos + mask) & ~mask
@@ -40,11 +49,14 @@
def appendobj(self, value):
self.value = value
- def get_buffer_as_string_maybe(self):
- return self.mr.input, self.mr.inputpos
+ def get_buffer_and_pos(self):
+ return self.mr.inputbuf, self.mr.inputpos
- def skip(self, size):
- self.read(size) # XXX, could avoid taking the slice
+ def can_advance(self, size):
+ return self.mr.can_advance(size)
+
+ def advance(self, size):
+ self.mr.advance(size)
ReaderForPos.__name__ = 'ReaderForPos%d' % pos
return ReaderForPos
diff --git a/rpython/rlib/rstruct/standardfmttable.py
b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -12,34 +12,84 @@
from rpython.rlib.rstruct import ieee
from rpython.rlib.rstruct.error import StructError, StructOverflowError
from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.strstorage import str_storage_getitem
+from rpython.rlib.buffer import StringBuffer
from rpython.rlib import rarithmetic
+from rpython.rlib.buffer import CannotRead, CannotWrite
from rpython.rtyper.lltypesystem import rffi
+USE_FASTPATH = True # set to False by some tests
+ALLOW_SLOWPATH = True # set to False by some tests
+ALLOW_FASTPATH = True # set to False by some tests
+
native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1)
native_is_ieee754 = float.__getformat__('double').startswith('IEEE')
[email protected]()
+def pack_fastpath(TYPE):
+ """
+ Create a fast path packer for TYPE. The packer returns True is it succeded
+ or False otherwise.
+ """
+ @specialize.argtype(0)
+ def do_pack_fastpath(fmtiter, value):
+ size = rffi.sizeof(TYPE)
+ if (not USE_FASTPATH or
+ fmtiter.bigendian != native_is_bigendian or
+ not native_is_ieee754):
+ raise CannotWrite
+ #
+ # typed_write() might raise CannotWrite
+ fmtiter.wbuf.typed_write(TYPE, fmtiter.pos, value)
+ if not ALLOW_FASTPATH:
+ # if we are here it means that typed_write did not raise, and thus
+ # the fast path was actually taken
+ raise ValueError("fastpath not allowed :(")
+ fmtiter.advance(size)
+ #
+ @specialize.argtype(0)
+ def do_pack_fastpath_maybe(fmtiter, value):
+ try:
+ do_pack_fastpath(fmtiter, value)
+ except CannotWrite:
+ if not ALLOW_SLOWPATH:
+ raise ValueError("fastpath not taken :(")
+ return False
+ else:
+ return True
+ #
+ return do_pack_fastpath_maybe
+
def pack_pad(fmtiter, count):
- fmtiter.result.append_multiple_char('\x00', count)
+ fmtiter.wbuf.setzeros(fmtiter.pos, count)
+ fmtiter.advance(count)
def pack_char(fmtiter):
string = fmtiter.accept_str_arg()
if len(string) != 1:
raise StructError("expected a string of length 1")
c = string[0] # string->char conversion for the annotator
- fmtiter.result.append(c)
+ fmtiter.wbuf.setitem(fmtiter.pos, c)
+ fmtiter.advance(1)
def pack_bool(fmtiter):
c = '\x01' if fmtiter.accept_bool_arg() else '\x00'
- fmtiter.result.append(c)
+ fmtiter.wbuf.setitem(fmtiter.pos, c)
+ fmtiter.advance(1)
+
+def _pack_string(fmtiter, string, count):
+ pos = fmtiter.pos
+ if len(string) < count:
+ n = len(string)
+ fmtiter.wbuf.setslice(pos, string)
+ fmtiter.wbuf.setzeros(pos+n, count-n)
+ else:
+ assert count >= 0
+ fmtiter.wbuf.setslice(pos, string[:count])
+ fmtiter.advance(count)
def pack_string(fmtiter, count):
string = fmtiter.accept_str_arg()
- if len(string) < count:
- fmtiter.result.append(string)
- fmtiter.result.append_multiple_char('\x00', count - len(string))
- else:
- fmtiter.result.append_slice(string, 0, count)
+ _pack_string(fmtiter, string, count)
def pack_pascal(fmtiter, count):
string = fmtiter.accept_str_arg()
@@ -49,21 +99,28 @@
if prefix < 0:
raise StructError("bad '0p' in struct format")
if prefix > 255:
- prefixchar = '\xff'
- else:
- prefixchar = chr(prefix)
- fmtiter.result.append(prefixchar)
- fmtiter.result.append_slice(string, 0, prefix)
- fmtiter.result.append_multiple_char('\x00', count - (1 + prefix))
+ prefix = 255
+ fmtiter.wbuf.setitem(fmtiter.pos, chr(prefix))
+ fmtiter.advance(1)
+ _pack_string(fmtiter, string, count-1)
-def make_float_packer(size):
+
+def make_float_packer(TYPE):
+ size = rffi.sizeof(TYPE)
def packer(fmtiter):
fl = fmtiter.accept_float_arg()
+ if TYPE is not rffi.FLOAT and pack_fastpath(TYPE)(fmtiter, fl):
+ return
+ # slow path
try:
- return ieee.pack_float(fmtiter.result, fl, size, fmtiter.bigendian)
+ result = ieee.pack_float(fmtiter.wbuf, fmtiter.pos,
+ fl, size, fmtiter.bigendian)
except OverflowError:
assert size == 4
raise StructOverflowError("float too large for format 'f'")
+ else:
+ fmtiter.advance(size)
+ return result
return packer
# ____________________________________________________________
@@ -111,42 +168,56 @@
errormsg = "argument out of range for %d-byte%s integer format" % (size,
plural)
unroll_revrange_size = unrolling_iterable(range(size-1, -1, -1))
+ TYPE = get_rffi_int_type(size, signed)
def pack_int(fmtiter):
method = getattr(fmtiter, accept_method)
value = method()
if not min <= value <= max:
raise StructError(errormsg)
+ #
+ if pack_fastpath(TYPE)(fmtiter, value):
+ return
+ #
+ pos = fmtiter.pos + size - 1
if fmtiter.bigendian:
for i in unroll_revrange_size:
x = (value >> (8*i)) & 0xff
- fmtiter.result.append(chr(x))
+ fmtiter.wbuf.setitem(pos-i, chr(x))
else:
+
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
