Alvaro Herrera wrote:
> > tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len);
> >
> > which fails because (HEAPTUPLESIZE + len) is again considered
> > an invalid size, the size being 1468006476 in my test.
>
> Um, it seems reasonable to make this one be a huge-zero-alloc:
>
> MemoryContextAllocExtended(CurrentMemoryContext,
> HEAPTUPLESIZE + len,
> MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO)
Good, this allows the tests to go to completion! The tests in question
are dump/reload of a row with several fields totalling 1.4GB (deflated),
with COPY TO/FROM file and psql's \copy in both directions, as well as
pg_dump followed by pg_restore|psql.
The modified patch is attached.
It provides a useful mitigation to dump/reload databases having
rows in the 1GB-2GB range, but it works under these limitations:
- no single field has a text representation exceeding 1GB.
- no row as text exceeds 2GB (\copy from fails beyond that. AFAICS we
could push this to 4GB with limited changes to libpq, by
interpreting the Int32 field in the CopyData message as unsigned).
It's also possible to go beyond 4GB per row with this patch, but
when not using the protocol. I've managed to get a 5.6GB single-row
file with COPY TO file. That doesn't help with pg_dump, but that might
be useful in other situations.
In StringInfo, I've changed int64 to Size, because on 32 bits platforms
the downcast from int64 to Size is problematic, and as the rest of the
allocation routines seems to favor Size, it seems more consistent
anyway.
I couldn't test on 32 bits though, as I seem to never have enough
free contiguous memory available on a 32 bits VM to handle
that kind of data.
Best regards,
--
Daniel Vérité
PostgreSQL-powered mailer: http://www.manitou-mail.org
Twitter: @DanielVerite
diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c
index 6d0f3f3..ed9cabd 100644
--- a/src/backend/access/common/heaptuple.c
+++ b/src/backend/access/common/heaptuple.c
@@ -741,7 +741,9 @@ heap_form_tuple(TupleDesc tupleDescriptor,
* Allocate and zero the space needed. Note that the tuple body and
* HeapTupleData management structure are allocated in one chunk.
*/
- tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len);
+ tuple = MemoryContextAllocExtended(CurrentMemoryContext,
+ HEAPTUPLESIZE + len,
+ MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO);
tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
/*
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 3201476..ce681d7 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -446,6 +446,15 @@ SendCopyEnd(CopyState cstate)
}
}
+/*
+ * Prepare for output
+ */
+static void
+CopyStartSend(CopyState cstate)
+{
+ allowLongStringInfo(cstate->fe_msgbuf);
+}
+
/*----------
* CopySendData sends output data to the destination (file or frontend)
* CopySendString does the same for null-terminated strings
@@ -2015,6 +2024,8 @@ CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls)
MemoryContextReset(cstate->rowcontext);
oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
+ CopyStartSend(cstate);
+
if (cstate->binary)
{
/* Binary per-tuple header */
@@ -3203,6 +3214,7 @@ CopyReadLine(CopyState cstate)
bool result;
resetStringInfo(&cstate->line_buf);
+ allowLongStringInfo(&cstate->line_buf);
cstate->line_buf_valid = true;
/* Mark that encoding conversion hasn't occurred yet */
@@ -3272,6 +3284,7 @@ CopyReadLine(CopyState cstate)
{
/* transfer converted data back to line_buf */
resetStringInfo(&cstate->line_buf);
+ allowLongStringInfo(&cstate->line_buf);
appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
pfree(cvt);
}
@@ -3696,7 +3709,7 @@ CopyReadAttributesText(CopyState cstate)
}
resetStringInfo(&cstate->attribute_buf);
-
+ allowLongStringInfo(&cstate->attribute_buf);
/*
* The de-escaped attributes will certainly not be longer than the input
* data line, so we can just force attribute_buf to be large enough and
diff --git a/src/backend/lib/stringinfo.c b/src/backend/lib/stringinfo.c
index 7382e08..6e451b2 100644
--- a/src/backend/lib/stringinfo.c
+++ b/src/backend/lib/stringinfo.c
@@ -47,12 +47,24 @@ initStringInfo(StringInfo str)
{
int size = 1024; /* initial default buffer size */
- str->data = (char *) palloc(size);
+ str->data = (char *) palloc(size); /* no need for "huge" at this point */
str->maxlen = size;
+ str->allowlong = false;
resetStringInfo(str);
}
/*
+ * allocLongStringInfo
+ *
+ * Mark the StringInfo as a candidate for a "huge" allocation
+ */
+void
+allowLongStringInfo(StringInfo str)
+{
+ str->allowlong = true;
+}
+
+/*
* resetStringInfo
*
* Reset the StringInfo: the data buffer remains valid, but its
@@ -64,6 +76,7 @@ resetStringInfo(StringInfo str)
str->data[0] = '\0';
str->len = 0;
str->cursor = 0;
+ str->allowlong = false;
}
/*
@@ -244,7 +257,9 @@ appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
void
enlargeStringInfo(StringInfo str, int needed)
{
- int newlen;
+ Size newlen;
+ Size total_needed;
+ Size limit;
/*
* Guard against out-of-range "needed" values. Without this, we can get
@@ -252,18 +267,21 @@ enlargeStringInfo(StringInfo str, int needed)
*/
if (needed < 0) /* should not happen */
elog(ERROR, "invalid string enlargement request size: %d", needed);
- if (((Size) needed) >= (MaxAllocSize - (Size) str->len))
+
+ /* choose the proper limit and verify this allocation wouldn't exceed it */
+ limit = str->allowlong ? MaxAllocHugeSize : MaxAllocSize;
+ if (((Size) needed) >= (limit - str->len))
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("out of memory"),
- errdetail("Cannot enlarge string buffer containing %d bytes by %d more bytes.",
- str->len, needed)));
+ errdetail("Cannot enlarge string buffer containing "INT64_FORMAT" bytes by %d more bytes.",
+ (int64)str->len, needed)));
- needed += str->len + 1; /* total space required now */
+ total_needed = needed + str->len + 1; /* total space required now */
- /* Because of the above test, we now have needed <= MaxAllocSize */
+ /* Because of the above test, we now have needed <= limit */
- if (needed <= str->maxlen)
+ if (total_needed <= str->maxlen)
return; /* got enough space already */
/*
@@ -272,18 +290,18 @@ enlargeStringInfo(StringInfo str, int needed)
* Actually, we might need to more than double it if 'needed' is big...
*/
newlen = 2 * str->maxlen;
- while (needed > newlen)
+ while (total_needed > newlen)
newlen = 2 * newlen;
/*
- * Clamp to MaxAllocSize in case we went past it. Note we are assuming
- * here that MaxAllocSize <= INT_MAX/2, else the above loop could
+ * Clamp to the limit in case we went past it. Note we are assuming
+ * here that MaxAllocHugeSize <= INT64_MAX/2, else the above loop could
* overflow. We will still have newlen >= needed.
*/
- if (newlen > (int) MaxAllocSize)
- newlen = (int) MaxAllocSize;
+ if (newlen > limit)
+ newlen = limit;
- str->data = (char *) repalloc(str->data, newlen);
+ str->data = (char *) repalloc_huge(str->data, newlen);
str->maxlen = newlen;
}
diff --git a/src/include/lib/stringinfo.h b/src/include/lib/stringinfo.h
index f644067..8e3f43b 100644
--- a/src/include/lib/stringinfo.h
+++ b/src/include/lib/stringinfo.h
@@ -30,14 +30,17 @@
* cursor is initialized to zero by makeStringInfo or initStringInfo,
* but is not otherwise touched by the stringinfo.c routines.
* Some routines use it to scan through a StringInfo.
+ * allowlong boolean flag indicating whether this StringInfo can allocate
+ * more than MaxAllocSize bytes.
*-------------------------
*/
typedef struct StringInfoData
{
char *data;
- int len;
- int maxlen;
+ Size len;
+ Size maxlen;
int cursor;
+ bool allowlong;
} StringInfoData;
typedef StringInfoData *StringInfo;
@@ -79,6 +82,11 @@ extern StringInfo makeStringInfo(void);
extern void initStringInfo(StringInfo str);
/*------------------------
+ * Set flag to allow "huge" stringinfos.
+ */
+extern void allowLongStringInfo(StringInfo str);
+
+/*------------------------
* resetStringInfo
* Clears the current content of the StringInfo, if any. The
* StringInfo remains valid.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
