Charles-François Natali <neolo...@free.fr> added the comment:
Here's a new version with a test (untested).
Note that I'm absolutely not sure that the 'memsize' argument to
bigmemtest is correct.
----------
Added file: http://bugs.python.org/file24079/pickle_overflow-2.diff
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue13555>
_______________________________________
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -6,7 +6,8 @@
import pickletools
import copy_reg
-from test.test_support import TestFailed, have_unicode, TESTFN
+from test.test_support import (TestFailed, have_unicode, TESTFN, _2G,
+ precisionbigmemtest)
# Tests that try a number of pickle protocols should have a
# for proto in protocols:
@@ -14,6 +15,8 @@
assert pickle.HIGHEST_PROTOCOL == cPickle.HIGHEST_PROTOCOL == 2
protocols = range(pickle.HIGHEST_PROTOCOL + 1)
+ascii_char_size = 1
+
# Copy of test.test_support.run_with_locale. This is needed to support Python
# 2.4, which didn't include it. This is all to support test_xpickle, which
# bounces pickled objects through older Python versions to test backwards
@@ -1280,3 +1283,21 @@
f.write(pickled2)
f.seek(0)
self.assertEqual(unpickler.load(), data2)
+
+class BigmemPickleTests(unittest.TestCase):
+
+ # All protocols use 1-byte per printable ASCII character; we add another
+ # byte because the encoded form has to be copied into the internal buffer.
+
+ @precisionbigmemtest(size=_2G, memuse=3 + ascii_char_size)
+ def test_huge_str_32b(self, size):
+ data = "abcd" * (size // 4)
+ try:
+ for proto in protocols:
+ try:
+ pickled = self.dumps(data, proto)
+ self.loads(pickled)
+ finally:
+ pickled = None
+ finally:
+ data = None
diff --git a/Lib/test/test_cpickle.py b/Lib/test/test_cpickle.py
--- a/Lib/test/test_cpickle.py
+++ b/Lib/test/test_cpickle.py
@@ -1,7 +1,9 @@
import cPickle, unittest
from cStringIO import StringIO
-from test.pickletester import AbstractPickleTests, AbstractPickleModuleTests
-from test.pickletester import AbstractPicklerUnpicklerObjectTests
+from test.pickletester import (AbstractPickleTests,
+ AbstractPickleModuleTests,
+ AbstractPicklerUnpicklerObjectTests,
+ BigmemPickleTests)
from test import test_support
class cPickleTests(AbstractPickleTests, AbstractPickleModuleTests):
@@ -101,6 +103,16 @@
pickler_class = cPickle.Pickler
unpickler_class = cPickle.Unpickler
+class cPickleBigmemPickleTests(BigmemPickleTests):
+
+ def dumps(self, arg, proto=0, fast=0):
+ # Ignore fast
+ return cPickle.dumps(arg, proto)
+
+ def loads(self, buf):
+ # Ignore fast
+ return cPickle.loads(buf)
+
class Node(object):
pass
@@ -133,6 +145,7 @@
cPickleFastPicklerTests,
cPickleDeepRecursive,
cPicklePicklerUnpicklerObjectTests,
+ cPickleBigmemPickleTests,
)
if __name__ == "__main__":
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -3,10 +3,11 @@
from test import test_support
-from test.pickletester import AbstractPickleTests
-from test.pickletester import AbstractPickleModuleTests
-from test.pickletester import AbstractPersistentPicklerTests
-from test.pickletester import AbstractPicklerUnpicklerObjectTests
+from test.pickletester import (AbstractPickleTests,
+ AbstractPickleModuleTests,
+ AbstractPersistentPicklerTests,
+ AbstractPicklerUnpicklerObjectTests,
+ BigmemPickleTests)
class PickleTests(AbstractPickleTests, AbstractPickleModuleTests):
@@ -66,6 +67,16 @@
pickler_class = pickle.Pickler
unpickler_class = pickle.Unpickler
+class PickleBigmemPickleTests(BigmemPickleTests):
+
+ def dumps(self, arg, proto=0, fast=0):
+ # Ignore fast
+ return pickle.dumps(arg, proto)
+
+ def loads(self, buf):
+ # Ignore fast
+ return pickle.loads(buf)
+
def test_main():
test_support.run_unittest(
@@ -73,6 +84,7 @@
PicklerTests,
PersPicklerTests,
PicklerUnpicklerObjectTests,
+ PickleBigmemPickleTests,
)
test_support.run_doctest(pickle)
diff --git a/Modules/cPickle.c b/Modules/cPickle.c
--- a/Modules/cPickle.c
+++ b/Modules/cPickle.c
@@ -139,15 +139,15 @@
typedef struct {
PyObject_HEAD
- int length; /* number of initial slots in data currently used */
- int size; /* number of slots in data allocated */
+ Py_ssize_t length; /* number of initial slots in data currently used */
+ Py_ssize_t size; /* number of slots in data allocated */
PyObject **data;
} Pdata;
static void
Pdata_dealloc(Pdata *self)
{
- int i;
+ Py_ssize_t i;
PyObject **p;
for (i = self->length, p = self->data; --i >= 0; p++) {
@@ -193,9 +193,9 @@
* number of items, this is a (non-erroneous) NOP.
*/
static int
-Pdata_clear(Pdata *self, int clearto)
+Pdata_clear(Pdata *self, Py_ssize_t clearto)
{
- int i;
+ Py_ssize_t i;
PyObject **p;
if (clearto < 0) return stackUnderflow();
@@ -214,18 +214,17 @@
static int
Pdata_grow(Pdata *self)
{
- int bigger;
- size_t nbytes;
+ Py_ssize_t bigger;
+ Py_ssize_t nbytes;
+
PyObject **tmp;
+ if (self->size > (PY_SSIZE_T_MAX >> 1))
+ goto nomemory;
bigger = self->size << 1;
- if (bigger <= 0) /* was 0, or new value overflows */
+ if (bigger > (PY_SSIZE_T_MAX / sizeof(PyObject *)))
goto nomemory;
- if ((int)(size_t)bigger != bigger)
- goto nomemory;
- nbytes = (size_t)bigger * sizeof(PyObject *);
- if (nbytes / sizeof(PyObject *) != (size_t)bigger)
- goto nomemory;
+ nbytes = bigger * sizeof(PyObject *);
tmp = realloc(self->data, nbytes);
if (tmp == NULL)
goto nomemory;
@@ -280,10 +279,10 @@
static PyObject *
-Pdata_popTuple(Pdata *self, int start)
+Pdata_popTuple(Pdata *self, Py_ssize_t start)
{
PyObject *r;
- int i, j, l;
+ Py_ssize_t i, j, l;
l = self->length-start;
r = PyTuple_New(l);
@@ -297,10 +296,10 @@
}
static PyObject *
-Pdata_popList(Pdata *self, int start)
+Pdata_popList(Pdata *self, Py_ssize_t start)
{
PyObject *r;
- int i, j, l;
+ Py_ssize_t i, j, l;
l=self->length-start;
if (!( r=PyList_New(l))) return NULL;
@@ -347,9 +346,9 @@
int bin;
int fast; /* Fast mode doesn't save in memo, don't use if circ ref */
- int (*write_func)(struct Picklerobject *, const char *, Py_ssize_t);
+ Py_ssize_t (*write_func)(struct Picklerobject *, const char *, Py_ssize_t);
char *write_buf;
- int buf_size;
+ Py_ssize_t buf_size;
PyObject *dispatch_table;
int fast_container; /* count nested container dumps */
PyObject *fast_memo;
@@ -373,12 +372,12 @@
PyObject *mark;
PyObject *pers_func;
PyObject *last_string;
- int *marks;
- int num_marks;
- int marks_size;
+ Py_ssize_t *marks;
+ Py_ssize_t num_marks;
+ Py_ssize_t marks_size;
Py_ssize_t (*read_func)(struct Unpicklerobject *, char **, Py_ssize_t);
Py_ssize_t (*readline_func)(struct Unpicklerobject *, char **);
- int buf_size;
+ Py_ssize_t buf_size;
char *buf;
PyObject *find_class;
} Unpicklerobject;
@@ -424,7 +423,7 @@
return NULL;
}
-static int
+static Py_ssize_t
write_file(Picklerobject *self, const char *s, Py_ssize_t n)
{
size_t nbyteswritten;
@@ -433,11 +432,6 @@
return 0;
}
- if (n > INT_MAX) {
- /* String too large */
- return -1;
- }
-
PyFile_IncUseCount((PyFileObject *)self->file);
Py_BEGIN_ALLOW_THREADS
nbyteswritten = fwrite(s, sizeof(char), n, self->fp);
@@ -448,10 +442,10 @@
return -1;
}
- return (int)n;
+ return n;
}
-static int
+static Py_ssize_t
write_cStringIO(Picklerobject *self, const char *s, Py_ssize_t n)
{
if (s == NULL) {
@@ -462,26 +456,21 @@
return -1;
}
- return (int)n;
+ return n;
}
-static int
+static Py_ssize_t
write_none(Picklerobject *self, const char *s, Py_ssize_t n)
{
if (s == NULL) return 0;
- if (n > INT_MAX) return -1;
- return (int)n;
+ return n;
}
-static int
-write_other(Picklerobject *self, const char *s, Py_ssize_t _n)
+static Py_ssize_t
+write_other(Picklerobject *self, const char *s, Py_ssize_t n)
{
PyObject *py_str = 0, *junk = 0;
- int n;
-
- if (_n > INT_MAX)
- return -1;
- n = (int)_n;
+
if (s == NULL) {
if (!( self->buf_size )) return 0;
py_str = PyString_FromStringAndSize(self->write_buf,
@@ -531,7 +520,7 @@
size_t nbytesread;
if (self->buf_size == 0) {
- int size;
+ Py_ssize_t size;
size = ((n < 32) ? 32 : n);
if (!( self->buf = (char *)malloc(size))) {
@@ -575,7 +564,7 @@
static Py_ssize_t
readline_file(Unpicklerobject *self, char **s)
{
- int i;
+ Py_ssize_t i;
if (self->buf_size == 0) {
if (!( self->buf = (char *)malloc(40))) {
@@ -587,7 +576,7 @@
i = 0;
while (1) {
- int bigger;
+ Py_ssize_t bigger;
char *newbuf;
for (; i < (self->buf_size - 1); i++) {
if (feof(self->fp) ||
@@ -597,13 +586,13 @@
return i + 1;
}
}
- bigger = self->buf_size << 1;
- if (bigger <= 0) { /* overflow */
+ if (self->buf_size < (PY_SSIZE_T_MAX >> 1)) {
PyErr_NoMemory();
return -1;
}
+ bigger = self->buf_size << 1;
newbuf = (char *)realloc(self->buf, bigger);
- if (!newbuf) {
+ if (newbuf == NULL) {
PyErr_NoMemory();
return -1;
}
@@ -700,7 +689,7 @@
* The caller is responsible for free()'ing the return value.
*/
static char *
-pystrndup(const char *s, int n)
+pystrndup(const char *s, Py_ssize_t n)
{
char *r = (char *)malloc(n+1);
if (r == NULL)
@@ -715,7 +704,7 @@
get(Picklerobject *self, PyObject *id)
{
PyObject *value, *mv;
- long c_value;
+ Py_ssize_t c_value;
char s[30];
size_t len;
@@ -735,7 +724,8 @@
if (!self->bin) {
s[0] = GET;
- PyOS_snprintf(s + 1, sizeof(s) - 1, "%ld\n", c_value);
+ PyOS_snprintf(s + 1, sizeof(s) - 1,
+ "%" PY_FORMAT_SIZE_T "d\n", c_value);
len = strlen(s);
}
else if (Pdata_Check(self->file)) {
@@ -780,8 +770,7 @@
put2(Picklerobject *self, PyObject *ob)
{
char c_str[30];
- int p;
- size_t len;
+ Py_ssize_t len, p;
int res = -1;
PyObject *py_ob_id = 0, *memo_len = 0, *t = 0;
@@ -994,7 +983,7 @@
{
char c_str[32];
long l = PyInt_AS_LONG((PyIntObject *)args);
- int len = 0;
+ Py_ssize_t len = 0;
if (!self->bin
#if SIZEOF_LONG > 4
@@ -1201,7 +1190,7 @@
static int
save_string(Picklerobject *self, PyObject *args, int doput)
{
- int size, len;
+ Py_ssize_t size, len;
PyObject *repr=0;
if ((size = PyString_Size(args)) < 0)
@@ -1448,7 +1437,7 @@
static int
store_tuple_elements(Picklerobject *self, PyObject *t, int len)
{
- int i;
+ Py_ssize_t i;
int res = -1; /* guilty until proved innocent */
assert(PyTuple_Size(t) == len);
@@ -1477,7 +1466,7 @@
save_tuple(Picklerobject *self, PyObject *args)
{
PyObject *py_tuple_id = NULL;
- int len, i;
+ Py_ssize_t len, i;
int res = -1;
static char tuple = TUPLE;
@@ -1690,7 +1679,7 @@
{
int res = -1;
char s[3];
- int len;
+ Py_ssize_t len;
PyObject *iter;
if (self->fast && !fast_save_enter(self, args))
@@ -1943,7 +1932,7 @@
{
int res = -1;
char s[3];
- int len;
+ Py_ssize_t len;
if (self->fast && !fast_save_enter(self, args))
goto finally;
@@ -2027,7 +2016,7 @@
if ((getinitargs_func = PyObject_GetAttr(args, __getinitargs___str))) {
PyObject *element = 0;
- int i, len;
+ Py_ssize_t i, len;
if (!( class_args =
PyObject_Call(getinitargs_func, empty_tuple, NULL)))
@@ -2289,7 +2278,8 @@
save_pers(Picklerobject *self, PyObject *args, PyObject *f)
{
PyObject *pid = 0;
- int size, res = -1;
+ Py_ssize_t size;
+ int res = -1;
static char persid = PERSID, binpersid = BINPERSID;
@@ -2431,7 +2421,7 @@
if (use_newobj) {
PyObject *cls;
PyObject *newargtup;
- int n, i;
+ Py_ssize_t n, i;
/* Sanity checks. */
n = PyTuple_Size(argtup);
@@ -2815,7 +2805,7 @@
static PyObject *
Pickle_getvalue(Picklerobject *self, PyObject *args)
{
- int l, i, rsize, ssize, clear=1, lm;
+ Py_ssize_t l, i, rsize, ssize, clear=1, lm;
long ik;
PyObject *k, *r;
char *s, *p, *have_get;
@@ -3314,7 +3304,7 @@
return global;
}
-static int
+static Py_ssize_t
marker(Unpicklerobject *self)
{
if (self->num_marks < 1) {
@@ -3345,7 +3335,8 @@
{
PyObject *py_int = 0;
char *endptr, *s;
- int len, res = -1;
+ Py_ssize_t len;
+ int res = -1;
long l;
if ((len = self->readline_func(self, &s)) < 0) return -1;
@@ -3477,7 +3468,8 @@
{
PyObject *l = 0;
char *end, *s;
- int len, res = -1;
+ Py_ssize_t len;
+ int res = -1;
if ((len = self->readline_func(self, &s)) < 0) return -1;
if (len < 2) return bad_readline();
@@ -3541,7 +3533,8 @@
{
PyObject *py_float = 0;
char *endptr, *s;
- int len, res = -1;
+ Py_ssize_t len;
+ int res = -1;
double d;
if ((len = self->readline_func(self, &s)) < 0) return -1;
@@ -3597,7 +3590,8 @@
load_string(Unpicklerobject *self)
{
PyObject *str = 0;
- int len, res = -1;
+ Py_ssize_t len;
+ int res = -1;
char *s, *p;
if ((len = self->readline_func(self, &s)) < 0) return -1;
@@ -3639,7 +3633,7 @@
load_binstring(Unpicklerobject *self)
{
PyObject *py_string = 0;
- long l;
+ Py_ssize_t l;
char *s;
if (self->read_func(self, &s, 4) < 0) return -1;
@@ -3691,20 +3685,17 @@
load_unicode(Unpicklerobject *self)
{
PyObject *str = 0;
- int len, res = -1;
+ Py_ssize_t len;
char *s;
if ((len = self->readline_func(self, &s)) < 0) return -1;
if (len < 1) return bad_readline();
if (!( str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL)))
- goto finally;
+ return -1;
PDATA_PUSH(self->stack, str, -1);
return 0;
-
- finally:
- return res;
}
#endif
@@ -3714,7 +3705,7 @@
load_binunicode(Unpicklerobject *self)
{
PyObject *unicode;
- long l;
+ Py_ssize_t l;
char *s;
if (self->read_func(self, &s, 4) < 0) return -1;
@@ -3745,7 +3736,7 @@
load_tuple(Unpicklerobject *self)
{
PyObject *tup;
- int i;
+ Py_ssize_t i;
if ((i = marker(self)) < 0) return -1;
if (!( tup=Pdata_popTuple(self->stack, i))) return -1;
@@ -3798,7 +3789,7 @@
load_list(Unpicklerobject *self)
{
PyObject *list = 0;
- int i;
+ Py_ssize_t i;
if ((i = marker(self)) < 0) return -1;
if (!( list=Pdata_popList(self->stack, i))) return -1;
@@ -3810,7 +3801,7 @@
load_dict(Unpicklerobject *self)
{
PyObject *dict, *key, *value;
- int i, j, k;
+ Py_ssize_t i, j, k;
if ((i = marker(self)) < 0) return -1;
j=self->stack->length;
@@ -3886,7 +3877,7 @@
load_obj(Unpicklerobject *self)
{
PyObject *class, *tup, *obj=0;
- int i;
+ Py_ssize_t i;
if ((i = marker(self)) < 0) return -1;
if (!( tup=Pdata_popTuple(self->stack, i+1))) return -1;
@@ -3907,7 +3898,7 @@
load_inst(Unpicklerobject *self)
{
PyObject *tup, *class=0, *obj=0, *module_name, *class_name;
- int i, len;
+ Py_ssize_t i, len;
char *s;
if ((i = marker(self)) < 0) return -1;
@@ -3993,7 +3984,7 @@
load_global(Unpicklerobject *self)
{
PyObject *class = 0, *module_name = 0, *class_name = 0;
- int len;
+ Py_ssize_t len;
char *s;
if ((len = self->readline_func(self, &s)) < 0) return -1;
@@ -4024,7 +4015,7 @@
load_persid(Unpicklerobject *self)
{
PyObject *pid = 0;
- int len;
+ Py_ssize_t len;
char *s;
if (self->pers_func) {
@@ -4102,7 +4093,7 @@
static int
load_pop(Unpicklerobject *self)
{
- int len = self->stack->length;
+ Py_ssize_t len = self->stack->length;
/* Note that we split the (pickle.py) stack into two stacks,
an object stack and a mark stack. We have to be clever and
@@ -4127,7 +4118,7 @@
static int
load_pop_mark(Unpicklerobject *self)
{
- int i;
+ Py_ssize_t i;
if ((i = marker(self)) < 0)
return -1;
@@ -4142,7 +4133,7 @@
load_dup(Unpicklerobject *self)
{
PyObject *last;
- int len;
+ Py_ssize_t len;
if ((len = self->stack->length) <= 0) return stackUnderflow();
last=self->stack->data[len-1];
@@ -4156,7 +4147,7 @@
load_get(Unpicklerobject *self)
{
PyObject *py_str = 0, *value = 0;
- int len;
+ Py_ssize_t len;
char *s;
int rc;
@@ -4214,7 +4205,7 @@
PyObject *py_key = 0, *value = 0;
unsigned char c;
char *s;
- long key;
+ Py_ssize_t key;
int rc;
if (self->read_func(self, &s, 4) < 0) return -1;
@@ -4317,7 +4308,7 @@
load_put(Unpicklerobject *self)
{
PyObject *py_str = 0, *value = 0;
- int len, l;
+ Py_ssize_t len, l;
char *s;
if ((l = self->readline_func(self, &s)) < 0) return -1;
@@ -4337,7 +4328,7 @@
PyObject *py_key = 0, *value = 0;
unsigned char key;
char *s;
- int len;
+ Py_ssize_t len;
if (self->read_func(self, &s, 1) < 0) return -1;
if (!( (len=self->stack->length) > 0 )) return stackUnderflow();
@@ -4356,10 +4347,10 @@
load_long_binput(Unpicklerobject *self)
{
PyObject *py_key = 0, *value = 0;
- long key;
+ Py_ssize_t key;
unsigned char c;
char *s;
- int len;
+ Py_ssize_t len;
if (self->read_func(self, &s, 4) < 0) return -1;
if (!( len=self->stack->length )) return stackUnderflow();
@@ -4382,10 +4373,10 @@
static int
-do_append(Unpicklerobject *self, int x)
+do_append(Unpicklerobject *self, Py_ssize_t x)
{
PyObject *value = 0, *list = 0, *append_method = 0;
- int len, i;
+ Py_ssize_t len, i;
len=self->stack->length;
if (!( len >= x && x > 0 )) return stackUnderflow();
@@ -4451,11 +4442,11 @@
}
-static int
-do_setitems(Unpicklerobject *self, int x)
+static Py_ssize_t
+do_setitems(Unpicklerobject *self, Py_ssize_t x)
{
PyObject *value = 0, *key = 0, *dict = 0;
- int len, i, r=0;
+ Py_ssize_t len, i, r=0;
if (!( (len=self->stack->length) >= x
&& x > 0 )) return stackUnderflow();
@@ -4496,8 +4487,8 @@
PyObject *state, *inst, *slotstate;
PyObject *__setstate__;
PyObject *d_key, *d_value;
+ int res = -1;
Py_ssize_t i;
- int res = -1;
/* Stack is ... instance, state. We want to leave instance at
* the stack top, possibly mutated via instance.__setstate__(state).
@@ -4596,7 +4587,7 @@
static int
load_mark(Unpicklerobject *self)
{
- int s;
+ Py_ssize_t s;
/* Note that we split the (pickle.py) stack into two stacks, an
object stack and a mark stack. Here we push a mark onto the
@@ -4981,7 +4972,7 @@
static int
noload_obj(Unpicklerobject *self)
{
- int i;
+ Py_ssize_t i;
if ((i = marker(self)) < 0) return -1;
return Pdata_clear(self->stack, i+1);
@@ -4991,7 +4982,7 @@
static int
noload_inst(Unpicklerobject *self)
{
- int i;
+ Py_ssize_t i;
char *s;
if ((i = marker(self)) < 0) return -1;
@@ -5068,7 +5059,7 @@
static int
noload_appends(Unpicklerobject *self)
{
- int i;
+ Py_ssize_t i;
if ((i = marker(self)) < 0) return -1;
return Pdata_clear(self->stack, i);
}
@@ -5082,7 +5073,7 @@
static int
noload_setitems(Unpicklerobject *self)
{
- int i;
+ Py_ssize_t i;
if ((i = marker(self)) < 0) return -1;
return Pdata_clear(self->stack, i);
}
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
http://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
