Index: Include/bytesobject.h =================================================================== --- Include/bytesobject.h (revision 58203) +++ Include/bytesobject.h (working copy) @@ -8,7 +8,7 @@ #include -/* Type PyBytesObject represents a mutable array of bytes. +/* Type PyBytesObject represents an immutable array of bytes. * The Python API is that of a sequence; * the bytes are mapped to ints in [0, 256). * Bytes are not characters; they may be used to encode characters. Index: Objects/bytesobject.c =================================================================== --- Objects/bytesobject.c (revision 58203) +++ Objects/bytesobject.c (working copy) @@ -32,24 +32,7 @@ /* end nullbytes support */ -/* Helpers */ - static int -_getbytevalue(PyObject* arg, int *value) -{ - PyObject *intarg = PyNumber_Int(arg); - if (! intarg) - return 0; - *value = PyInt_AsLong(intarg); - Py_DECREF(intarg); - if (*value < 0 || *value >= 256) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return 0; - } - return 1; -} - -static int bytes_getbuffer(PyBytesObject *obj, PyBuffer *view, int flags) { int ret; @@ -62,7 +45,8 @@ ptr = ""; else ptr = obj->ob_bytes; - ret = PyBuffer_FillInfo(view, ptr, Py_Size(obj), 0, flags); + ret = PyBuffer_FillInfo(view, ptr, Py_Size(obj), + 1 /* readonly */, flags); if (ret >= 0) { obj->ob_exports++; } @@ -262,7 +246,7 @@ } static PyObject * -bytes_iconcat(PyBytesObject *self, PyObject *other) +_bytes_iconcat(PyBytesObject *self, PyObject *other) { Py_ssize_t mysize; Py_ssize_t size; @@ -320,37 +304,6 @@ return (PyObject *)result; } -static PyObject * -bytes_irepeat(PyBytesObject *self, Py_ssize_t count) -{ - Py_ssize_t mysize; - Py_ssize_t size; - - if (count < 0) - count = 0; - mysize = Py_Size(self); - size = mysize * count; - if (count != 0 && size / count != mysize) - return PyErr_NoMemory(); - if (size < self->ob_alloc) { - Py_Size(self) = size; - self->ob_bytes[Py_Size(self)] = '\0'; /* Trailing null byte */ - } - else if (PyBytes_Resize((PyObject *)self, size) < 0) - return NULL; - - if (mysize == 1) - memset(self->ob_bytes, self->ob_bytes[0], size); - else { - Py_ssize_t i; - for (i = 1; i < count; i++) - memcpy(self->ob_bytes + i*mysize, self->ob_bytes, mysize); - } - - Py_INCREF(self); - return (PyObject *)self; -} - static int bytes_substring(PyBytesObject *self, PyBytesObject *other) { @@ -457,279 +410,6 @@ } static int -bytes_setslice(PyBytesObject *self, Py_ssize_t lo, Py_ssize_t hi, - PyObject *values) -{ - Py_ssize_t avail, needed; - void *bytes; - PyBuffer vbytes; - int res = 0; - - vbytes.len = -1; - if (values == (PyObject *)self) { - /* Make a copy and call this function recursively */ - int err; - values = PyBytes_FromObject(values); - if (values == NULL) - return -1; - err = bytes_setslice(self, lo, hi, values); - Py_DECREF(values); - return err; - } - if (values == NULL) { - /* del b[lo:hi] */ - bytes = NULL; - needed = 0; - } - else { - if (_getbuffer(values, &vbytes) < 0) { - PyErr_Format(PyExc_TypeError, - "can't set bytes slice from %.100s", - Py_Type(values)->tp_name); - return -1; - } - needed = vbytes.len; - bytes = vbytes.buf; - } - - if (lo < 0) - lo = 0; - if (hi < lo) - hi = lo; - if (hi > Py_Size(self)) - hi = Py_Size(self); - - avail = hi - lo; - if (avail < 0) - lo = hi = avail = 0; - - if (avail != needed) { - if (avail > needed) { - /* - 0 lo hi old_size - | |<----avail----->|<-----tomove------>| - | |<-needed->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, - Py_Size(self) - hi); - } - /* XXX(nnorwitz): need to verify this can't overflow! */ - if (PyBytes_Resize((PyObject *)self, - Py_Size(self) + needed - avail) < 0) { - res = -1; - goto finish; - } - if (avail < needed) { - /* - 0 lo hi old_size - | |<-avail->|<-----tomove------>| - | |<----needed---->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, - Py_Size(self) - lo - needed); - } - } - - if (needed > 0) - memcpy(self->ob_bytes + lo, bytes, needed); - - - finish: - if (vbytes.len != -1) - PyObject_ReleaseBuffer(values, &vbytes); - return res; -} - -static int -bytes_setitem(PyBytesObject *self, Py_ssize_t i, PyObject *value) -{ - Py_ssize_t ival; - - if (i < 0) - i += Py_Size(self); - - if (i < 0 || i >= Py_Size(self)) { - PyErr_SetString(PyExc_IndexError, "bytes index out of range"); - return -1; - } - - if (value == NULL) - return bytes_setslice(self, i, i+1, NULL); - - ival = PyNumber_AsSsize_t(value, PyExc_ValueError); - if (ival == -1 && PyErr_Occurred()) - return -1; - - if (ival < 0 || ival >= 256) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return -1; - } - - self->ob_bytes[i] = ival; - return 0; -} - -static int -bytes_ass_subscript(PyBytesObject *self, PyObject *item, PyObject *values) -{ - Py_ssize_t start, stop, step, slicelen, needed; - char *bytes; - - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - - if (i == -1 && PyErr_Occurred()) - return -1; - - if (i < 0) - i += PyBytes_GET_SIZE(self); - - if (i < 0 || i >= Py_Size(self)) { - PyErr_SetString(PyExc_IndexError, "bytes index out of range"); - return -1; - } - - if (values == NULL) { - /* Fall through to slice assignment */ - start = i; - stop = i + 1; - step = 1; - slicelen = 1; - } - else { - Py_ssize_t ival = PyNumber_AsSsize_t(values, PyExc_ValueError); - if (ival == -1 && PyErr_Occurred()) - return -1; - if (ival < 0 || ival >= 256) { - PyErr_SetString(PyExc_ValueError, - "byte must be in range(0, 256)"); - return -1; - } - self->ob_bytes[i] = (char)ival; - return 0; - } - } - else if (PySlice_Check(item)) { - if (PySlice_GetIndicesEx((PySliceObject *)item, - PyBytes_GET_SIZE(self), - &start, &stop, &step, &slicelen) < 0) { - return -1; - } - } - else { - PyErr_SetString(PyExc_TypeError, "bytes indices must be integer"); - return -1; - } - - if (values == NULL) { - bytes = NULL; - needed = 0; - } - else if (values == (PyObject *)self || !PyBytes_Check(values)) { - /* Make a copy an call this function recursively */ - int err; - values = PyBytes_FromObject(values); - if (values == NULL) - return -1; - err = bytes_ass_subscript(self, item, values); - Py_DECREF(values); - return err; - } - else { - assert(PyBytes_Check(values)); - bytes = ((PyBytesObject *)values)->ob_bytes; - needed = Py_Size(values); - } - /* Make sure b[5:2] = ... inserts before 5, not before 2. */ - if ((step < 0 && start < stop) || - (step > 0 && start > stop)) - stop = start; - if (step == 1) { - if (slicelen != needed) { - if (slicelen > needed) { - /* - 0 start stop old_size - | |<---slicelen--->|<-----tomove------>| - | |<-needed->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, - Py_Size(self) - stop); - } - if (PyBytes_Resize((PyObject *)self, - Py_Size(self) + needed - slicelen) < 0) - return -1; - if (slicelen < needed) { - /* - 0 lo hi old_size - | |<-avail->|<-----tomove------>| - | |<----needed---->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, - Py_Size(self) - start - needed); - } - } - - if (needed > 0) - memcpy(self->ob_bytes + start, bytes, needed); - - return 0; - } - else { - if (needed == 0) { - /* Delete slice */ - Py_ssize_t cur, i; - - if (step < 0) { - stop = start + 1; - start = stop + step * (slicelen - 1) - 1; - step = -step; - } - for (cur = start, i = 0; - i < slicelen; cur += step, i++) { - Py_ssize_t lim = step - 1; - - if (cur + step >= PyBytes_GET_SIZE(self)) - lim = PyBytes_GET_SIZE(self) - cur - 1; - - memmove(self->ob_bytes + cur - i, - self->ob_bytes + cur + 1, lim); - } - /* Move the tail of the bytes, in one chunk */ - cur = start + slicelen*step; - if (cur < PyBytes_GET_SIZE(self)) { - memmove(self->ob_bytes + cur - slicelen, - self->ob_bytes + cur, - PyBytes_GET_SIZE(self) - cur); - } - if (PyBytes_Resize((PyObject *)self, - PyBytes_GET_SIZE(self) - slicelen) < 0) - return -1; - - return 0; - } - else { - /* Assign slice */ - Py_ssize_t cur, i; - - if (needed != slicelen) { - PyErr_Format(PyExc_ValueError, - "attempt to assign bytes of size %zd " - "to extended slice of size %zd", - needed, slicelen); - return -1; - } - for (cur = start, i = 0; i < slicelen; cur += step, i++) - self->ob_bytes[cur] = bytes[i]; - return 0; - } - } -} - -static int bytes_init(PyBytesObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"source", "encoding", "errors", 0}; @@ -779,7 +459,7 @@ Py_DECREF(encoded); return -1; } - new = bytes_iconcat(self, encoded); + new = _bytes_iconcat(self, encoded); Py_DECREF(encoded); if (new == NULL) return -1; @@ -2451,169 +2131,6 @@ return NULL; } -PyDoc_STRVAR(extend__doc__, -"B.extend(iterable int) -> None\n\ -\n\ -Append all the elements from the iterator or sequence to the\n\ -end of the bytes."); -static PyObject * -bytes_extend(PyBytesObject *self, PyObject *arg) -{ - if (bytes_setslice(self, Py_Size(self), Py_Size(self), arg) == -1) - return NULL; - Py_RETURN_NONE; -} - - -PyDoc_STRVAR(reverse__doc__, -"B.reverse() -> None\n\ -\n\ -Reverse the order of the values in bytes in place."); -static PyObject * -bytes_reverse(PyBytesObject *self, PyObject *unused) -{ - char swap, *head, *tail; - Py_ssize_t i, j, n = Py_Size(self); - - j = n / 2; - head = self->ob_bytes; - tail = head + n - 1; - for (i = 0; i < j; i++) { - swap = *head; - *head++ = *tail; - *tail-- = swap; - } - - Py_RETURN_NONE; -} - -PyDoc_STRVAR(insert__doc__, -"B.insert(index, int) -> None\n\ -\n\ -Insert a single item into the bytes before the given index."); -static PyObject * -bytes_insert(PyBytesObject *self, PyObject *args) -{ - int value; - Py_ssize_t where, n = Py_Size(self); - - if (!PyArg_ParseTuple(args, "ni:insert", &where, &value)) - return NULL; - - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytes"); - return NULL; - } - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "byte must be in range(0, 256)"); - return NULL; - } - if (PyBytes_Resize((PyObject *)self, n + 1) < 0) - return NULL; - - if (where < 0) { - where += n; - if (where < 0) - where = 0; - } - if (where > n) - where = n; - memmove(self->ob_bytes + where + 1, self->ob_bytes + where, n - where); - self->ob_bytes[where] = value; - - Py_RETURN_NONE; -} - -PyDoc_STRVAR(append__doc__, -"B.append(int) -> None\n\ -\n\ -Append a single item to the end of the bytes."); -static PyObject * -bytes_append(PyBytesObject *self, PyObject *arg) -{ - int value; - Py_ssize_t n = Py_Size(self); - - if (! _getbytevalue(arg, &value)) - return NULL; - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytes"); - return NULL; - } - if (PyBytes_Resize((PyObject *)self, n + 1) < 0) - return NULL; - - self->ob_bytes[n] = value; - - Py_RETURN_NONE; -} - -PyDoc_STRVAR(pop__doc__, -"B.pop([index]) -> int\n\ -\n\ -Remove and return a single item from the bytes. If no index\n\ -argument is give, will pop the last value."); -static PyObject * -bytes_pop(PyBytesObject *self, PyObject *args) -{ - int value; - Py_ssize_t where = -1, n = Py_Size(self); - - if (!PyArg_ParseTuple(args, "|n:pop", &where)) - return NULL; - - if (n == 0) { - PyErr_SetString(PyExc_OverflowError, - "cannot pop an empty bytes"); - return NULL; - } - if (where < 0) - where += Py_Size(self); - if (where < 0 || where >= Py_Size(self)) { - PyErr_SetString(PyExc_IndexError, "pop index out of range"); - return NULL; - } - - value = self->ob_bytes[where]; - memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); - if (PyBytes_Resize((PyObject *)self, n - 1) < 0) - return NULL; - - return PyInt_FromLong(value); -} - -PyDoc_STRVAR(remove__doc__, -"B.remove(int) -> None\n\ -\n\ -Remove the first occurance of a value in bytes"); -static PyObject * -bytes_remove(PyBytesObject *self, PyObject *arg) -{ - int value; - Py_ssize_t where, n = Py_Size(self); - - if (! _getbytevalue(arg, &value)) - return NULL; - - for (where = 0; where < n; where++) { - if (self->ob_bytes[where] == value) - break; - } - if (where == n) { - PyErr_SetString(PyExc_ValueError, "value not found in bytes"); - return NULL; - } - - memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); - if (PyBytes_Resize((PyObject *)self, n - 1) < 0) - return NULL; - - Py_RETURN_NONE; -} - /* XXX These two helpers could be optimized if argsize == 1 */ static Py_ssize_t @@ -2927,17 +2444,17 @@ (ssizeargfunc)bytes_repeat, /* sq_repeat */ (ssizeargfunc)bytes_getitem, /* sq_item */ 0, /* sq_slice */ - (ssizeobjargproc)bytes_setitem, /* sq_ass_item */ + 0, /* sq_ass_item */ 0, /* sq_ass_slice */ (objobjproc)bytes_contains, /* sq_contains */ - (binaryfunc)bytes_iconcat, /* sq_inplace_concat */ - (ssizeargfunc)bytes_irepeat, /* sq_inplace_repeat */ + 0, /* sq_inplace_concat */ + 0, /* sq_inplace_repeat */ }; static PyMappingMethods bytes_as_mapping = { (lenfunc)bytes_length, (binaryfunc)bytes_subscript, - (objobjargproc)bytes_ass_subscript, + 0, }; static PyBufferProcs bytes_as_buffer = { @@ -2961,12 +2478,6 @@ {"rpartition", (PyCFunction)bytes_rpartition, METH_O, rpartition__doc__}, {"split", (PyCFunction)bytes_split, METH_VARARGS, split__doc__}, {"rsplit", (PyCFunction)bytes_rsplit, METH_VARARGS, rsplit__doc__}, - {"extend", (PyCFunction)bytes_extend, METH_O, extend__doc__}, - {"insert", (PyCFunction)bytes_insert, METH_VARARGS, insert__doc__}, - {"append", (PyCFunction)bytes_append, METH_O, append__doc__}, - {"reverse", (PyCFunction)bytes_reverse, METH_NOARGS, reverse__doc__}, - {"pop", (PyCFunction)bytes_pop, METH_VARARGS, pop__doc__}, - {"remove", (PyCFunction)bytes_remove, METH_O, remove__doc__}, {"strip", (PyCFunction)bytes_strip, METH_VARARGS, strip__doc__}, {"lstrip", (PyCFunction)bytes_lstrip, METH_VARARGS, lstrip__doc__}, {"rstrip", (PyCFunction)bytes_rstrip, METH_VARARGS, rstrip__doc__}, @@ -2998,7 +2509,7 @@ 0, /* tp_as_number */ &bytes_as_sequence, /* tp_as_sequence */ &bytes_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ + 0, /* tp_hash TODO(jyasskin): define this.*/ 0, /* tp_call */ (reprfunc)bytes_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ Index: Lib/io.py =================================================================== --- Lib/io.py (revision 58203) +++ Lib/io.py (working copy) @@ -29,12 +29,13 @@ "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper"] -import os +import _fileio import abc -import sys +import array import codecs -import _fileio import io +import os +import sys import warnings # open() uses st_blksize whenever we can @@ -438,10 +439,10 @@ n = -1 if n < 0: return self.readall() - b = bytes(n.__index__()) - n = self.readinto(b) - del b[n:] - return b + buf = array.array('B', bytes(n.__index__())) + n = self.readinto(buf) + del buf[n:] + return bytes(buf) def readall(self): """readall() -> bytes. Read until EOF, using multiple read() call.""" @@ -634,14 +635,14 @@ # XXX More docs def __init__(self, initial_bytes=None): - buffer = b"" + buffer = array.array('B') if initial_bytes is not None: - buffer += initial_bytes + buffer += array.array('B', initial_bytes) self._buffer = buffer self._pos = 0 def getvalue(self): - return self._buffer + return bytes(self._buffer) def read(self, n=None): if n is None: @@ -651,7 +652,7 @@ newpos = min(len(self._buffer), self._pos + n) b = self._buffer[self._pos : newpos] self._pos = newpos - return b + return bytes(b) def read1(self, n): return self.read(n) @@ -666,9 +667,10 @@ if newpos > len(self._buffer): # Inserts null bytes between the current end of the file # and the new write position. - padding = b'\x00' * (newpos - len(self._buffer) - n) + padding = (array.array('B', b'\x00') * + (newpos - len(self._buffer) - n)) self._buffer[self._pos:newpos - n] = padding - self._buffer[self._pos:newpos] = b + self._buffer[self._pos:newpos] = array.array('B', b) self._pos = newpos return n @@ -815,7 +817,7 @@ # XXX Why not just let the exception pass through? raise BlockingIOError(e.errno, e.strerror, 0) before = len(self._write_buf) - self._write_buf.extend(b) + self._write_buf += b written = len(self._write_buf) - before if len(self._write_buf) > self.buffer_size: try: @@ -836,11 +838,11 @@ try: while self._write_buf: n = self.raw.write(self._write_buf) - del self._write_buf[:n] + self._write_buf = self._write_buf[n:] written += n except BlockingIOError as e: n = e.characters_written - del self._write_buf[:n] + self._write_buf = self._write_buf[n:] written += n raise BlockingIOError(e.errno, e.strerror, written) @@ -1138,11 +1140,11 @@ x, pos = divmod(pos, 1<<64) if not x: return None, pos - b = b"" - while x: - b.append(x&0xff) - x >>= 8 - return str(b[::-1]), pos + def reversed_bytes_in_x(): + while x: + yield x&0xff + x >>= 8 + return str(reversed(reversed_bytes_in_x())), pos def tell(self): if not self._seekable: @@ -1165,7 +1167,7 @@ try: decoder.setstate((b"", decoder_state)) n = 0 - bb = bytes(1) + bb = array.array('B', b'\0') for i, bb[0] in enumerate(readahead): n += len(decoder.decode(bb)) if n >= needed: Index: Modules/Setup.dist =================================================================== --- Modules/Setup.dist (revision 58203) +++ Modules/Setup.dist (working copy) @@ -153,7 +153,7 @@ # Modules that should always be present (non UNIX dependent): -#array arraymodule.c # array objects +array arraymodule.c # array objects #cmath cmathmodule.c # -lm # complex math library functions #math mathmodule.c # -lm # math library functions, e.g. sin() #_struct _struct.c # binary structure packing/unpacking