I wrote: > Yes, this refactoring was good for testing actually... Oops, I have been too hasty when sending previous patch, there was a bug related to huge allocations. Patch correcting this bug is attached.
Attached are as well two things I have used to test the new API:
- A hack refactoring the existing routines MemoryContextAlloc* to use
the extended API
- An extension with a function doing a direct call to the extended API
able to control the flags used:
CREATE FUNCTION blackhole_palloc(size bigint,
is_huge bool,
is_no_oom bool,
is_zero bool,
is_zero_aligned bool)
Here are some tests done on a small box of 384MB with direct calls of
the extended API:
=# create extension blackhole ;
CREATE EXTENSION
-- failure for normal allocation because size >= 1GB
=# select blackhole_palloc(1024 * 1024 * 1024, false, false, false, false);
ERROR: XX000: invalid memory alloc request size 1073741824
LOCATION: MemoryContextAllocExtended, mcxt.c:628
-- Failure of OOM because normal allocation can be done, but no memory
=# select blackhole_palloc(1024 * 1024 * 1024 - 1, false, false, false, false);
ERROR: 53200: out of memory
DETAIL: Failed on request of size 1073741823.
LOCATION: MemoryContextAllocExtended, mcxt.c:639
-- No failure, bypassing OOM error
=# select blackhole_palloc(1024 * 1024 * 1024 - 1, false, true, false, false);
blackhole_palloc
------------------
null
(1 row)
-- Huge allocation, no error because OOM error is bypassed
=# select blackhole_palloc(1024 * 1024 * 1024, true, true, false, false);
blackhole_palloc
------------------
null
(1 row)
-- OOM error, huge allocation failure
=# select blackhole_palloc(1024 * 1024 * 1024, true, false, false, false);
ERROR: 53200: out of memory
DETAIL: Failed on request of size 1073741824.
LOCATION: MemoryContextAllocExtended, mcxt.c:639
-- Assertion failure, zero and zero aligned cannot be called at the same time
=# select blackhole_palloc(1024 * 1024, false, false, true, true);
server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
The connection to the server was lost. Attempting reset: Failed.
--
Michael
From 09bf9364a80dfe6426c91bdefe6ae2135e6f20c3 Mon Sep 17 00:00:00 2001
From: Michael Paquier <mich...@otacoo.com>
Date: Fri, 30 Jan 2015 12:56:21 +0900
Subject: [PATCH 1/2] Create MemoryContextAllocExtended central routine for
memory allocation
This new routine is the central point can be used by extensions and
third-part utilities in a more extensive way than the already present
routines MemoryContextAlloc, one of the control flags introduced being
particularly useful to avoid out-of-memory errors when allocation request
cannot be completed correctly.
---
src/backend/utils/mmgr/mcxt.c | 57 +++++++++++++++++++++++++++++++++++++++++++
src/include/utils/palloc.h | 12 +++++++++
2 files changed, 69 insertions(+)
diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c
index c62922a..fce8a0e 100644
--- a/src/backend/utils/mmgr/mcxt.c
+++ b/src/backend/utils/mmgr/mcxt.c
@@ -603,6 +603,63 @@ MemoryContextCreate(NodeTag tag, Size size,
}
/*
+ * MemoryContextAllocExtended
+ * Allocate space within the specified context using flag options
+ * defined by caller.
+ *
+ * The following control flags can be used:
+ * - MCXT_ALLOC_HUGE, allocate possibly-expansive space. this is
+ * equivalent to MemoryContextAllocHuge.
+ * - MCXT_ALLOC_NO_OOM, not fail in case of allocation request
+ * failure and return NULL.
+ * - MCXT_ALLOC_ZERO, clear allocated memory using MemSetAligned.
+ * - MCXT_ALLOC_ZERO_ALIGNED, clear memory using MemSetLoop.
+ */
+void *
+MemoryContextAllocExtended(MemoryContext context, Size size, int flags)
+{
+ void *ret;
+
+ AssertArg(MemoryContextIsValid(context));
+ AssertNotInCriticalSection(context);
+
+ if (((flags & MCXT_ALLOC_HUGE) != 0 && !AllocHugeSizeIsValid(size)) ||
+ ((flags & MCXT_ALLOC_HUGE) == 0 && !AllocSizeIsValid(size)))
+ elog(ERROR, "invalid memory alloc request size %zu", size);
+
+ context->isReset = false;
+
+ ret = (*context->methods->alloc) (context, size);
+ if ((flags & MCXT_ALLOC_NO_OOM) == 0 && ret == NULL)
+ {
+ MemoryContextStats(TopMemoryContext);
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory"),
+ errdetail("Failed on request of size %zu.", size)));
+ }
+
+ if (ret == NULL)
+ return NULL;
+
+ VALGRIND_MEMPOOL_ALLOC(context, ret, size);
+
+ /*
+ * MemSetAligned and MemSetLoop should not be called in the same
+ * context (see c.h for more details).
+ */
+ Assert((flags & MCXT_ALLOC_ZERO) == 0 ||
+ (flags & MCXT_ALLOC_ZERO_ALIGNED) == 0);
+
+ if ((flags & MCXT_ALLOC_ZERO) != 0)
+ MemSetAligned(ret, 0, size);
+ if ((flags & MCXT_ALLOC_ZERO_ALIGNED) != 0)
+ MemSetLoop(ret, 0, size);
+
+ return ret;
+}
+
+/*
* MemoryContextAlloc
* Allocate space within the specified context.
*
diff --git a/src/include/utils/palloc.h b/src/include/utils/palloc.h
index ca03f2b..34acabe 100644
--- a/src/include/utils/palloc.h
+++ b/src/include/utils/palloc.h
@@ -43,8 +43,20 @@ typedef struct MemoryContextData *MemoryContext;
extern PGDLLIMPORT MemoryContext CurrentMemoryContext;
/*
+ * Control flags for options of MemoryContextAllocExtended()
+ */
+#define MCXT_ALLOC_HUGE 0x01 /* huge allocation */
+#define MCXT_ALLOC_NO_OOM 0x02 /* no failure if out-of-memory */
+#define MCXT_ALLOC_ZERO 0x04 /* clear allocated memory using
+ * MemSetAligned */
+#define MCXT_ALLOC_ZERO_ALIGNED 0x08 /* clear allocated memory using
+ * MemSetLoop */
+
+/*
* Fundamental memory-allocation operations (more are in utils/memutils.h)
*/
+extern void *MemoryContextAllocExtended(MemoryContext context,
+ Size size, int flags);
extern void *MemoryContextAlloc(MemoryContext context, Size size);
extern void *MemoryContextAllocZero(MemoryContext context, Size size);
extern void *MemoryContextAllocZeroAligned(MemoryContext context, Size size);
--
2.2.2
From 88f3fb0129a6fad35114902320b006183ea6da5f Mon Sep 17 00:00:00 2001
From: Michael Paquier <mich...@otacoo.com>
Date: Fri, 30 Jan 2015 13:14:23 +0900
Subject: [PATCH 2/2] Small hack to test extended function for mcxt
---
src/backend/utils/mmgr/mcxt.c | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c
index fce8a0e..efa81fc 100644
--- a/src/backend/utils/mmgr/mcxt.c
+++ b/src/backend/utils/mmgr/mcxt.c
@@ -669,6 +669,8 @@ MemoryContextAllocExtended(MemoryContext context, Size size, int flags)
void *
MemoryContextAlloc(MemoryContext context, Size size)
{
+ return MemoryContextAllocExtended(context, size, 0);
+#ifdef NOT_USED
void *ret;
AssertArg(MemoryContextIsValid(context));
@@ -692,6 +694,7 @@ MemoryContextAlloc(MemoryContext context, Size size)
VALGRIND_MEMPOOL_ALLOC(context, ret, size);
return ret;
+#endif
}
/*
@@ -704,6 +707,8 @@ MemoryContextAlloc(MemoryContext context, Size size)
void *
MemoryContextAllocZero(MemoryContext context, Size size)
{
+ return MemoryContextAllocExtended(context, size, MCXT_ALLOC_ZERO);
+#ifdef NOT_USED
void *ret;
AssertArg(MemoryContextIsValid(context));
@@ -729,6 +734,7 @@ MemoryContextAllocZero(MemoryContext context, Size size)
MemSetAligned(ret, 0, size);
return ret;
+#endif
}
/*
@@ -741,6 +747,8 @@ MemoryContextAllocZero(MemoryContext context, Size size)
void *
MemoryContextAllocZeroAligned(MemoryContext context, Size size)
{
+ return MemoryContextAllocExtended(context, size, MCXT_ALLOC_ZERO_ALIGNED);
+#ifdef NOT_USED
void *ret;
AssertArg(MemoryContextIsValid(context));
@@ -766,6 +774,7 @@ MemoryContextAllocZeroAligned(MemoryContext context, Size size)
MemSetLoop(ret, 0, size);
return ret;
+#endif
}
void *
@@ -911,6 +920,8 @@ repalloc(void *pointer, Size size)
void *
MemoryContextAllocHuge(MemoryContext context, Size size)
{
+ return MemoryContextAllocExtended(context, size, MCXT_ALLOC_HUGE);
+#ifdef NOT_USED
void *ret;
AssertArg(MemoryContextIsValid(context));
@@ -934,6 +945,7 @@ MemoryContextAllocHuge(MemoryContext context, Size size)
VALGRIND_MEMPOOL_ALLOC(context, ret, size);
return ret;
+#endif
}
/*
--
2.2.2
blackhole.tar.gz
Description: GNU Zip compressed data
-- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
