From 2c59401a617c2e8f0fd07d4b02df45c46ce8f83f Mon Sep 17 00:00:00 2001 From: Maxim Orlov Date: Thu, 3 Nov 2022 15:33:50 +0300 Subject: [PATCH v1 1/2] palloc_aligned --- src/backend/utils/mmgr/Makefile | 1 + src/backend/utils/mmgr/alignedalloc.c | 93 ++++++++++++++++++ src/backend/utils/mmgr/mcxt.c | 114 +++++++++++++++++++++-- src/backend/utils/mmgr/meson.build | 1 + src/include/utils/memutils_internal.h | 13 ++- src/include/utils/memutils_memorychunk.h | 2 +- src/include/utils/palloc.h | 3 + 7 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 src/backend/utils/mmgr/alignedalloc.c diff --git a/src/backend/utils/mmgr/Makefile b/src/backend/utils/mmgr/Makefile index 3b4cfdbd52..dae3432c98 100644 --- a/src/backend/utils/mmgr/Makefile +++ b/src/backend/utils/mmgr/Makefile @@ -13,6 +13,7 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global OBJS = \ + alignedalloc.o \ aset.o \ dsa.o \ freepage.o \ diff --git a/src/backend/utils/mmgr/alignedalloc.c b/src/backend/utils/mmgr/alignedalloc.c new file mode 100644 index 0000000000..e581772758 --- /dev/null +++ b/src/backend/utils/mmgr/alignedalloc.c @@ -0,0 +1,93 @@ +/*------------------------------------------------------------------------- + * + * alignedalloc.c + * Allocator functions to implement palloc_aligned + * + * This is not a fully fledged MemoryContext type as there is no means to + * create a MemoryContext of this type. The code here only serves to allow + * operations such as pfree() and repalloc() to work correctly on a memory + * chunk that was allocated by palloc_align(). + * + * Portions Copyright (c) 2017-2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/mmgr/alignedalloc.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "utils/memdebug.h" +#include "utils/memutils_memorychunk.h" + +void +AlignedAllocFree(void *pointer) +{ + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + void *unaligned; + + Assert(!MemoryChunkIsExternal(chunk)); + + unaligned = MemoryChunkGetBlock(chunk); + + pfree(unaligned); +} + +void * +AlignedAllocRealloc(void *pointer, Size size) +{ + MemoryChunk *redirchunk = PointerGetMemoryChunk(pointer); + Size alignto = MemoryChunkGetValue(redirchunk); + void *unaligned = MemoryChunkGetBlock(redirchunk); + MemoryChunk *chunk = PointerGetMemoryChunk(unaligned); + Size old_size; + void *newptr; + + /* sanity check this is a power of 2 value */ + Assert((alignto & (alignto - 1)) == 0); + + /* + * Determine the size of the original allocation. We can't determine this + * exactly as GetMemoryChunkSpace() returns the total space used for the + * allocation, which for contexts like aset includes rounding up to the + * next power of 2. However, this value is just used to memcpy() the old + * data into the new allocation, so we only need to concern ourselves with + * not reading beyond the end of the original allocation's memory. The + * drawback here is that we may copy more bytes than we need to, which + * amounts only to wasted effort. + */ + old_size = GetMemoryChunkSpace(unaligned) - + ((char *)pointer - (char *)chunk); + + newptr = palloc_aligned(size, alignto, 0); + + /* + * We may memcpy beyond the end of the orignal allocation request size, so + * we must mark the entire allocation as defined. + */ + VALGRIND_MAKE_MEM_DEFINED(pointer, old_size); + memcpy(newptr, pointer, Min(size, old_size)); + pfree(unaligned); + + return newptr; +} + +MemoryContext +AlignedAllocGetChunkContext(void *pointer) +{ + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + + Assert(!MemoryChunkIsExternal(chunk)); + + return GetMemoryChunkContext(MemoryChunkGetBlock(chunk)); +} + +Size +AlignedGetChunkSpace(void *pointer) +{ + MemoryChunk *redirchunk = PointerGetMemoryChunk(pointer); + void *unaligned = MemoryChunkGetBlock(redirchunk); + + return GetMemoryChunkSpace(unaligned); +} diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index f526ca82c1..cd2c43efb3 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -30,6 +30,7 @@ #include "utils/memdebug.h" #include "utils/memutils.h" #include "utils/memutils_internal.h" +#include "utils/memutils_memorychunk.h" static void BogusFree(void *pointer); @@ -84,6 +85,21 @@ static const MemoryContextMethods mcxt_methods[] = { [MCTX_SLAB_ID].check = SlabCheck, #endif + /* alignedalloc.c */ + [MCTX_ALIGNED_REDIRECT_ID].alloc = NULL, /* not required */ + [MCTX_ALIGNED_REDIRECT_ID].free_p = AlignedAllocFree, + [MCTX_ALIGNED_REDIRECT_ID].realloc = AlignedAllocRealloc, + [MCTX_ALIGNED_REDIRECT_ID].reset = NULL, /* not required */ + [MCTX_ALIGNED_REDIRECT_ID].delete_context = NULL, /* not required */ + [MCTX_ALIGNED_REDIRECT_ID].get_chunk_context = AlignedAllocGetChunkContext, + [MCTX_ALIGNED_REDIRECT_ID].get_chunk_space = AlignedGetChunkSpace, + [MCTX_ALIGNED_REDIRECT_ID].is_empty = NULL, /* not required */ + [MCTX_ALIGNED_REDIRECT_ID].stats = NULL, /* not required */ +#ifdef MEMORY_CONTEXT_CHECKING + [MCTX_ALIGNED_REDIRECT_ID].check = NULL, /* no required */ +#endif + + /* * Unused (as yet) IDs should have dummy entries here. This allows us to * fail cleanly if a bogus pointer is passed to pfree or the like. It @@ -110,11 +126,6 @@ static const MemoryContextMethods mcxt_methods[] = { [MCTX_UNUSED4_ID].realloc = BogusRealloc, [MCTX_UNUSED4_ID].get_chunk_context = BogusGetChunkContext, [MCTX_UNUSED4_ID].get_chunk_space = BogusGetChunkSpace, - - [MCTX_UNUSED5_ID].free_p = BogusFree, - [MCTX_UNUSED5_ID].realloc = BogusRealloc, - [MCTX_UNUSED5_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED5_ID].get_chunk_space = BogusGetChunkSpace, }; /* @@ -1298,6 +1309,92 @@ palloc_extended(Size size, int flags) return ret; } +/* + * MemoryContextAllocAligned + * Allocate 'size' bytes of memory in 'context' aligned to 'alignto' + * bytes. + * + * 'flags' may be 0 or set the same as MemoryContextAllocExtended(). + * 'alignto' must be a power of 2. + */ +void * +MemoryContextAllocAligned(MemoryContext context, + Size size, Size alignto, int flags) +{ + Size alloc_size; + void *unaligned; + void *aligned; + + /* wouldn't make much sense to waste that much space */ + Assert(alignto < (128 * 1024 * 1024)); + + /* ensure alignto is a power of 2 */ + Assert((alignto & (alignto - 1)) == 0); + + /* + * If the alignment requirements are less than what we already guarantee + * then just use the standard allocation function. + */ + if (unlikely(alignto <= MAXIMUM_ALIGNOF)) + return palloc_extended(size, flags); + + /* + * We implement aligned pointers by simply allocating enough memory for + * the requested size plus the alignment and an additional MemoryChunk. + * This additional MemoryChunk is required for operations such as pfree + * when used on the pointer returned by this function. We use this + * "redirection" MemoryChunk in order to find the pointer to the memory + * that was returned by the MemoryContextAllocExtended call below. We do + * that by "borrowing" the block offset field and instead of using that to + * find the offset into the owning block, we use it to find the original + * allocated address. + * + * Here we must allocate enough extra memory so that we can still align + * the pointer returned by MemoryContextAllocExtended and also have enough + * space for the redirection MemoryChunk. + */ + alloc_size = size + alignto + sizeof(MemoryChunk); + + /* perform the actual allocation */ + unaligned = MemoryContextAllocExtended(context, alloc_size, flags); + + /* set the aligned pointer */ + aligned = (void *) TYPEALIGN(alignto, (char *) unaligned + sizeof(MemoryChunk)); + + /* + * We set the redirect MemoryChunk so that the block offset calculation is + * used to point back to the 'unaligned' allocated chunk. This allows us + * to use MemoryChunkGetBlock() to find the unaligned chunk when we need + * to perform operations such as pfree() or repalloc(). + * + * We store 'alignto' in the MemoryChunk's 'value' so that we know what + * the alignment was set to should we ever be asked to realloc this + * pointer. + */ + MemoryChunkSetHdrMask(PointerGetMemoryChunk(aligned), unaligned, alignto, + MCTX_ALIGNED_REDIRECT_ID); + + /* XXX: should we adjust valgrind state here? */ + + /* double check we produced a correctly aligned pointer */ + Assert((char *) TYPEALIGN(alignto, aligned) == aligned); + + return aligned; +} + +/* + * palloc_aligned + * Allocate 'size' bytes returning a pointer that's aligned to the + * 'alignto' boundary. + * + * 'alignto' must be a power of 2. + */ +void * +palloc_aligned(Size size, Size alignto, int flags) +{ + return MemoryContextAllocAligned(CurrentMemoryContext, size, alignto, flags); +} + /* * pfree * Release an allocated chunk. @@ -1306,11 +1403,16 @@ void pfree(void *pointer) { #ifdef USE_VALGRIND + MemoryContextMethodID method = GetMemoryChunkMethodID(pointer); MemoryContext context = GetMemoryChunkContext(pointer); #endif MCXT_METHOD(pointer, free_p) (pointer); - VALGRIND_MEMPOOL_FREE(context, pointer); + +#ifdef USE_VALGRIND + if (method != MCTX_ALIGNED_REDIRECT_ID) + VALGRIND_MEMPOOL_FREE(context, pointer); +#endif } /* diff --git a/src/backend/utils/mmgr/meson.build b/src/backend/utils/mmgr/meson.build index 641bb181ba..7cf4d6dcc8 100644 --- a/src/backend/utils/mmgr/meson.build +++ b/src/backend/utils/mmgr/meson.build @@ -1,4 +1,5 @@ backend_sources += files( + 'alignedalloc.c', 'aset.c', 'dsa.c', 'freepage.c', diff --git a/src/include/utils/memutils_internal.h b/src/include/utils/memutils_internal.h index bc2cbdd506..450bcba3ed 100644 --- a/src/include/utils/memutils_internal.h +++ b/src/include/utils/memutils_internal.h @@ -70,6 +70,15 @@ extern void SlabStats(MemoryContext context, extern void SlabCheck(MemoryContext context); #endif +/* + * These functions support the implementation of palloc_aligned() and are not + * part of a fully-fledged MemoryContext type. + */ +extern void AlignedAllocFree(void *pointer); +extern void *AlignedAllocRealloc(void *pointer, Size size); +extern MemoryContext AlignedAllocGetChunkContext(void *pointer); +extern Size AlignedGetChunkSpace(void *pointer); + /* * MemoryContextMethodID * A unique identifier for each MemoryContext implementation which @@ -92,8 +101,8 @@ typedef enum MemoryContextMethodID MCTX_ASET_ID, MCTX_GENERATION_ID, MCTX_SLAB_ID, - MCTX_UNUSED4_ID, /* available */ - MCTX_UNUSED5_ID /* 111 occurs in wipe_mem'd memory */ + MCTX_ALIGNED_REDIRECT_ID, + MCTX_UNUSED4_ID /* 111 occurs in wipe_mem'd memory */ } MemoryContextMethodID; /* diff --git a/src/include/utils/memutils_memorychunk.h b/src/include/utils/memutils_memorychunk.h index 2eefc138e3..38702efc58 100644 --- a/src/include/utils/memutils_memorychunk.h +++ b/src/include/utils/memutils_memorychunk.h @@ -156,7 +156,7 @@ MemoryChunkSetHdrMask(MemoryChunk *chunk, void *block, { Size blockoffset = (char *) chunk - (char *) block; - Assert((char *) chunk > (char *) block); + Assert((char *) chunk >= (char *) block); Assert(blockoffset <= MEMORYCHUNK_MAX_BLOCKOFFSET); Assert(value <= MEMORYCHUNK_MAX_VALUE); Assert((int) methodid <= MEMORY_CONTEXT_METHODID_MASK); diff --git a/src/include/utils/palloc.h b/src/include/utils/palloc.h index 8eee0e2938..989ddf18ef 100644 --- a/src/include/utils/palloc.h +++ b/src/include/utils/palloc.h @@ -73,10 +73,13 @@ extern void *MemoryContextAllocZero(MemoryContext context, Size size); extern void *MemoryContextAllocZeroAligned(MemoryContext context, Size size); extern void *MemoryContextAllocExtended(MemoryContext context, Size size, int flags); +extern void *MemoryContextAllocAligned(MemoryContext context, + Size size, Size alignto, int flags); extern void *palloc(Size size); extern void *palloc0(Size size); extern void *palloc_extended(Size size, int flags); +extern void *palloc_aligned(Size size, Size alignto, int flags); extern pg_nodiscard void *repalloc(void *pointer, Size size); extern pg_nodiscard void *repalloc_extended(void *pointer, Size size, int flags); -- 2.37.0 (Apple Git-136)