From 67018b04b7e11ec0f0644afbbd451f5fbaf0a6d6 Mon Sep 17 00:00:00 2001 From: kommih Date: Wed, 29 Aug 2018 13:52:39 +1000 Subject: [PATCH 1/3] copy memory limit fix To limit memory used by the COPY FROM because of slotification, calculates the tuple size of the first tuple in the batch and use that for remaining batch, so that it almost averages the memory usage by the COPY command. --- src/backend/commands/copy.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index c9272b344a..a82389b1a8 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -2314,6 +2314,7 @@ CopyFrom(CopyState cstate) #define MAX_BUFFERED_TUPLES 1000 TupleTableSlot **bufferedSlots = NULL; /* initialize to silence warning */ + Size bufferedSlotsSize = 0; uint64 firstBufferedLineNo = 0; Assert(cstate->rel); @@ -2753,24 +2754,26 @@ CopyFrom(CopyState cstate) /* Add this tuple to the tuple buffer */ if (nBufferedTuples == 0) firstBufferedLineNo = cstate->cur_lineno; + Assert(bufferedSlots[nBufferedTuples] == myslot); nBufferedTuples++; + bufferedSlotsSize += cstate->line_buf.len; /* * If the buffer filled up, flush it. Also flush if the * total size of all the tuples in the buffer becomes * large, to avoid using large amounts of memory for the * buffer when the tuples are exceptionally wide. - * - * PBORKED: Re-introduce size limit */ - if (nBufferedTuples == MAX_BUFFERED_TUPLES) + if (nBufferedTuples == MAX_BUFFERED_TUPLES || + bufferedSlotsSize > 65535) { CopyFromInsertBatch(cstate, estate, mycid, hi_options, resultRelInfo, bistate, nBufferedTuples, bufferedSlots, firstBufferedLineNo); nBufferedTuples = 0; + bufferedSlotsSize = 0; } } else -- 2.18.0.windows.1