diff --git a/src/backend/utils/mmgr/Makefile b/src/backend/utils/mmgr/Makefile index f644c40..07cf346 100644 --- a/src/backend/utils/mmgr/Makefile +++ b/src/backend/utils/mmgr/Makefile @@ -12,6 +12,7 @@ subdir = src/backend/utils/mmgr top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = aset.o dsa.o freepage.o generation.o mcxt.o memdebug.o portalmem.o slab.o +OBJS = aset.o dsa.o freepage.o generation.o mcxt.o memdebug.o portalmem.o slab.o \ + shm_retail.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index d52bd2c..3bb122f 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -397,6 +397,42 @@ MemoryContextSetParent(MemoryContext context, MemoryContext new_parent) } /* + * MemoryContextClone + * Create a new MemoryContext of the same type as template context + * and set its parent to specified one. + */ +MemoryContext +MemoryContextClone(MemoryContext template, MemoryContext parent) +{ + MemoryContext context; + + AssertArg(MemoryContextIsValid(template)); + AssertArg(MemoryContextIsValid(parent)); + + + switch (nodeTag(template)) + { + case T_AllocSetContext: + case T_SlabContext: + case T_GenerationContext: + elog(FATAL, " MemoryContextClone doesn't support cloning" + "memory context (%s)", template->name); + break; + case T_ShmRetailContext: + context = ShmRetailContextCreateLocal(parent, + "clone", + template); + break; + default: + elog(FATAL, "unrecognized node type: %d", + (int) nodeTag(template)); + } + return context; +} + + + +/* * MemoryContextAllowInCriticalSection * Allow/disallow allocations in this memory context within a critical * section. diff --git a/src/backend/utils/mmgr/shm_retail.c b/src/backend/utils/mmgr/shm_retail.c new file mode 100644 index 0000000..42d3569 --- /dev/null +++ b/src/backend/utils/mmgr/shm_retail.c @@ -0,0 +1,540 @@ +/*------------------------------------------------------------------------- + * + * shm_retail.c + * ShmRetailContext allocator definitions. + * + * ShmRetailContext is a MemoryContext implementation designed for cases where + * you want to allocate and free objects in the shared memory by MemoryContext + * API (palloc/pfree). The main use case is global system catalog cache. + * + * Portions Copyright (c) 2017-2019, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/mmgr/shm_retail.c + * + * + * NOTE: + * + * ShmRetailContext allows allocation in the shared memory via palloc(). + * This is intended to migrate locally allocated objects into shared memory. + * These objects could be syscache or somthing else. They usually + * allocate memory in local heap by palloc(). To take advantage of exisiting + * code, ShmRetailContext uses dsa_allocate()/dsa_free() as palloc()/pfree(). + * However, dsa_allocate() returns dsa_pointer while palloc() returns native + * pointer. And also an object may be a graph structure with pointers. + * It needs to remember either native pointer or dsa_pointer. + * + * So allow the creation of DSA areas inside the traditional fixed memory + * segment (instead of DSM), in a fixed-sized space reserved by the postmaster. + * In this case, dsa_pointer is directly castable to a raw pointer, which is + * common to every process. This fits to regular MemoryContext interface. But + * note that the total size is fixed at start up time. + * + * Now we can put objects into shared memory via palloc(). But without + * some garbage collection mechanism, memory leaks will be cluster-wide + * and cluster-life-time. Leaked object won't go away even if one backend + * exits. + * + * To address this issue, ShmRetailContext has two types of context: "local" and + * "global" one. Local context itself is located in local memory but chunks + * are allocated in shared memory. These chunks are pointed by chunk-list via + * dsa_pointer in order to free them all at once at transaction rollback. + * Once chunks are registerd to some index strucure in shared memory so that + * memory leak won't happen, you can move chunks to global ShmRetailContext, + * which is located in shared memory. + * + * Example of usage is found in: + * src/test/modules/test_shm_retail + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/ilist.h" +#include "storage/shmem.h" +#include "utils/dsa.h" +#include "utils/memdebug.h" +#include "utils/memutils.h" + + +/* + * An arbitrary number for how many chunks is pointed by one chunk list + * Use case like global system catalog cache calls palloc() in couple of + * times in one memory context. So 16 would be enough. + */ +#define NUM_CHUNKS 16 + +typedef struct chunk_list chunk_list; + +/* + * ShmRetailContext is a specialized memory context supporting dsa area created + * above Postmaster-initialized shared memory + */ +typedef struct ShmRetailContext +{ + MemoryContextData header; /* Standard memory-context fields */ + + /* ShmRetailContext parameters */ + void *base; /* raw address of Postmaster-initialized shared memory */ + dsa_area *area; /* dsa area created-in-place */ + /* array of pointers to chunks. If ShmRetailContext is permanent, NULL */ + chunk_list *chunk_list; +} ShmRetailContext; + +/* + * chunk_list + * keeping dsa_pointer to chunks to free them at rollback + * + * Local ShmRetailContext have this list while permanent one does not. + * If list becomes full, next list is pushed to head. + */ +struct chunk_list +{ + chunk_list *next; /* single linked list */ + int tail; /* index of array to be allocated */ + dsa_pointer chunks[NUM_CHUNKS]; /* relative address of chunks */ +}; + + +#define isLocalShmRetailContext(shm_context) (shm_context->chunk_list != NULL) +#define isListFull(list) (list->tail == NUM_CHUNKS) +#define dsaptr_to_rawptr(dsa_p, base_p) \ + ((char *)(base_p) + dsa_p) +#define rawptr_to_dsaptr(raw_p, base_p) \ + ((dsa_pointer) ((char *)raw_p - (char *)base_p)) + + + /* Helper function for ShmRetailContext API */ +static inline MemoryContext +ShmRetailContextCreateInternal(MemoryContext parent, + const char *name, + dsa_area *area, + void *base, + bool isLocal); +static void push_chunk_list(chunk_list *list); + + +/* + * These functions implement the MemoryContext API for ShmRetailContext. + */ +static void *ShmRetailContextAlloc(MemoryContext context, Size size); +static void ShmRetailContextFree(MemoryContext context, void *pointer); +static void *ShmRetailContextRealloc(MemoryContext context, void *pointer, Size size); +static void ShmRetailContextReset(MemoryContext context); +static void ShmRetailContextDelete(MemoryContext context); +static Size ShmRetailContextGetChunkSpace(MemoryContext context, void *pointer); +static bool ShmRetailContextIsEmpty(MemoryContext context); +static void ShmRetailContextStats(MemoryContext context, + MemoryStatsPrintFunc printfunc, void *passthru, + MemoryContextCounters *totals); +#ifdef MEMORY_CONTEXT_CHECKING +static void ShmRetailContextCheck(MemoryContext context); +#endif + +/* + * This is the virtual function table for ShmRetailContext contexts. + */ +static const MemoryContextMethods ShmRetailContextMethods = { + ShmRetailContextAlloc, + ShmRetailContextFree, + ShmRetailContextRealloc, + ShmRetailContextReset, + ShmRetailContextDelete, + ShmRetailContextGetChunkSpace, + ShmRetailContextIsEmpty, + ShmRetailContextStats +#ifdef MEMORY_CONTEXT_CHECKING + ,ShmRetailContextCheck +#endif +}; + + +/* + * ShmRetailContextCreateGlobal + * Create a new permanent ShmRetailContext context. + * + * parent: parent context, or NULL if top-level context + * name: name of context (must be statically allocated) + * area: dsa_area created in place of Postmaster-initialized shared memory + * base: address of Postmaster-initialized shared memory + * + * This context itself is allocated on shared memory. + */ +MemoryContext +ShmRetailContextCreateGlobal(MemoryContext parent, + const char *name, + dsa_area *area, + void *base) +{ + return ShmRetailContextCreateInternal(parent, name, area, base, false); +} + +/* + * CreateTempShmRetailContext + * Create local ShmRetailContext in local heap by ShmRetailContextCreate + * + * parent: parent context, or NULL if top-level context + * name: name of context (must be statically allocated) + * global_context: permanent shared MemoryContext + * + * Temp context inherits dsa_area and base address of permanent ShmRetailContext. + * This context itself is allocated on the parent context, which must not + * be permanent ShmRetailContext. + * + */ +MemoryContext +ShmRetailContextCreateLocal(MemoryContext parent, + const char *name, + MemoryContext global_context) +{ + ShmRetailContext *shmContext; + + AssertArg(MemoryContextIsValid(global_context)); + + shmContext = (ShmRetailContext *) global_context; + + /* global_context should be permanent one */ + Assert(!isLocalShmRetailContext(shmContext)); + + return ShmRetailContextCreateInternal(parent, name, + shmContext->area, shmContext->base, true); +} + + +/* + * ShmRetailContextCreateInternal + * Work-horse for ShmRetailContextCreateGlobal/ShmRetailContextCreateLocal + * + * parent: parent context, or NULL if top-level context + * name: name of context (must be statically allocated) + * area: dsa_area created in place of Postmaster-initialized shared memory + * base: address of Postmaster-initialized shared memory + * isLocal: context is local? + * + */ +static inline MemoryContext +ShmRetailContextCreateInternal(MemoryContext parent, + const char *name, + dsa_area *area, + void *base, + bool isLocal) +{ + ShmRetailContext *shmContext; + bool found; + + /* + * If context is temp, allocate it and its list in parent context. + * If it is permanent, chunk_list is not used. + */ + if (isLocal) + { + MemoryContext old_context; + + if (!parent) + elog(ERROR, "Parent context of local shared context" + "is not specified"); + + old_context = MemoryContextSwitchTo(parent); + + shmContext = palloc0(sizeof(ShmRetailContext)); + shmContext->chunk_list = (chunk_list *) + palloc0(sizeof(chunk_list)); + + MemoryContextSwitchTo(old_context); + } + else + { + shmContext = (ShmRetailContext *) + ShmemInitStruct(name, ShmRetailContextSize(), &found); + shmContext->chunk_list = NULL; + + if (found) + return &shmContext->header; + } + + MemoryContextCreate(&shmContext->header, + T_ShmRetailContext, + &ShmRetailContextMethods, + parent, + name); + + shmContext->base = base; + shmContext->area = area; + + return &shmContext->header; +} + +/* + * ShmRetailContextSize + * Size of ShmRetailContext + * If you use ShmRetailContextCreateGlobal, this fucntion should be called + * to reserve the size of ShmRetailContext at postgres start-up time + */ +Size +ShmRetailContextSize(void) +{ + return sizeof(ShmRetailContext); +} + + +/* + * ShmRetailContextMoveChunk + * + * We don't want to leak memory in shared memory. Unlike local process, + * memory leak still exists even after local process is terminated. + * If error occurs in transaction, we free all dsa_allocated chunks linked + * from local ShmRetailContext. When you make sure that the memory leak does + * not happen, ShmRetailContextMoveChunk should be called. + * + */ +void +ShmRetailContextMoveChunk(MemoryContext local_context, MemoryContext global_context) +{ + ShmRetailContext *localShmRetailContext = (ShmRetailContext *) local_context; + chunk_list *list = localShmRetailContext->chunk_list; + int idx; + + /* change backpointer to shared MemoryContext */ + while (list && list->chunks) + { + chunk_list *next_list = list->next; + + for (idx = 0; idx < list->tail; idx++) + { + /* Rewind to the secret start of the chunk */ + *(void **)(list->chunks[idx] + (char *)localShmRetailContext->base) + = global_context; + } + /* initialize tail of list */ + list->tail = 0; + list = next_list; + } +} + +/* + * push_chunk_list + * If chunk_list becomes full, add new list + */ +static void +push_chunk_list(chunk_list *list) +{ + MemoryContext old_context; + chunk_list *new_list; + + /* choose the same context as current list to make lifetime consistent */ + old_context = MemoryContextSwitchTo(GetMemoryChunkContext(list)); + + /* insert a new list into head position */ + new_list = (chunk_list *) palloc0(sizeof(chunk_list)); + new_list->next = list; + list = new_list; + + MemoryContextSwitchTo(old_context); +} + + +/* + * ShmRetailContextReset + * Free all the memory registered in chunk_list and list + * + * The chunks registered in chunk_list are dsa_freed. + * This does not affect permanent context. + * + */ +static void +ShmRetailContextReset(MemoryContext context) +{ + int idx; + chunk_list *list; + ShmRetailContext *shmContext = (ShmRetailContext *) context; + + Assert(shmContext); + + /* We don't support reset if context is permanent */ + if (!isLocalShmRetailContext(shmContext)) + elog(ERROR, + "reset is not supported at global ShmRetailContext"); + + + list = shmContext->chunk_list; + +#ifdef MEMORY_CONTEXT_CHECKING + /* Check for corruption and leaks before freeing */ + ShmRetailContextCheck(context); +#endif + + /* free all chunks and lists */ + while (list && list->chunks) + { + chunk_list *next_list = list->next; + + for (idx = 0; idx < list->tail; idx++) + dsa_free(shmContext->area, list->chunks[idx]); + + pfree(list); + list = next_list; + } +} + +/* + * ShmRetailContextDelete + * Free all the memory registered in chunk_list and context. + * See ShmRetailContextReset. + */ +static void +ShmRetailContextDelete(MemoryContext context) +{ + /* Reset to release all the chunk_lists */ + ShmRetailContextReset(context); + /* And free the context header */ + pfree(context); +} + +/* + * ShmRetailContextAlloc + * Returns native pointer to allocated memory + */ +static void * +ShmRetailContextAlloc(MemoryContext context, Size size) +{ + ShmRetailContext *shmContext = (ShmRetailContext *) context; + + char *chunk_backp; + chunk_list *list; + + /* we only allow palloc in local ShmRetailContext */ + if (!isLocalShmRetailContext(shmContext)) + { + elog(ERROR, "ShmRetailContextAlloc should be run in " + "local ShmRetailContext"); + return NULL; + } + + /* if list is full, allocate a new list */ + if (isListFull(shmContext->chunk_list)) + push_chunk_list(shmContext->chunk_list); + + /* Add space for the secret context pointer. */ + list = shmContext->chunk_list; + list->chunks[list->tail] = dsa_allocate(shmContext->area, + sizeof(void *) + size); + + chunk_backp = dsaptr_to_rawptr(list->chunks[list->tail], + shmContext->base); + + *(void **) chunk_backp = context; + list->tail++; + + return chunk_backp + sizeof(void *); +} + + +/* + * ShmRetailContextFree + * Frees allocated memory + */ +static void +ShmRetailContextFree(MemoryContext context, void *pointer) +{ + ShmRetailContext *shmContext = (ShmRetailContext *) context; + chunk_list *list; + char *chunk_backp; + dsa_pointer dp; + int idx; + + /* Rewind to the secret start of the chunk */ + chunk_backp = (char *) pointer - sizeof(void *); + + dp = rawptr_to_dsaptr(chunk_backp, shmContext->base); + + dsa_free(shmContext->area, dp); + + /* If global, no need to delete its reference from chunk_list */ + if (!isLocalShmRetailContext(shmContext)) + return; + + /* To avoid double free by ShmRetailContextDelete, remove its reference */ + for (list = shmContext->chunk_list; list != NULL; list = list->next) + { + for (idx = 0; idx < list->tail; idx++) + { + if (list->chunks[idx] == dp) + { + if (idx != list->tail - 1) + list->chunks[idx] = list->chunks[list->tail - 1]; + + list->tail--; + break; + } + } + } +} + +/* + * ShmRetailContextRealloc + * + * realloc() is not supported + */ +static void * +ShmRetailContextRealloc(MemoryContext context, void *pointer, Size size) +{ + elog(ERROR, "ShmRetailContext does not support realloc()"); + return NULL; /* keep compiler quiet */ +} + +/* + * ShmRetailContextGetChunkSpace + * Given a currently-allocated chunk, determine the total space + * it occupies (including all memory-allocation overhead). + */ +static Size +ShmRetailContextGetChunkSpace(MemoryContext context, void *pointer) +{ + elog(ERROR, "ShmRetailContext does not support get_chunk_space()"); + return 0; /* keep compiler quiet */ +} + +/* + * ShmRetailContextIsEmpty + * Is an ShmRetailContext empty of any allocated space? + */ +static bool +ShmRetailContextIsEmpty(MemoryContext context) +{ + elog(ERROR, "ShmRetailContext does not support is_empty()"); + return false; /* keep compiler quiet */ +} + +/* + * ShmRetailContextStats + * Compute stats about memory consumption of a ShmRetailContext context. + * + * XXX: can dsa_dump be used? + * printfunc: if not NULL, pass a human-readable stats string to this. + * passthru: pass this pointer through to printfunc. + * totals: if not NULL, add stats about this context into *totals. + */ +static void +ShmRetailContextStats(MemoryContext context, + MemoryStatsPrintFunc printfunc, void *passthru, + MemoryContextCounters *totals) +{ + elog(ERROR, "ShmRetailContext does not support stats()"); +} + + +#ifdef MEMORY_CONTEXT_CHECKING + +/* + * ShmRetailContextCheck + * + * XXX: for now, do nothing + */ +static void +ShmRetailContextCheck(MemoryContext context) +{ + +} + +#endif /* MEMORY_CONTEXT_CHECKING */ diff --git a/src/include/nodes/memnodes.h b/src/include/nodes/memnodes.h index 854bd5d..426e974 100644 --- a/src/include/nodes/memnodes.h +++ b/src/include/nodes/memnodes.h @@ -103,6 +103,7 @@ typedef struct MemoryContextData ((context) != NULL && \ (IsA((context), AllocSetContext) || \ IsA((context), SlabContext) || \ - IsA((context), GenerationContext))) + IsA((context), GenerationContext) || \ + IsA((context), ShmRetailContext))) #endif /* MEMNODES_H */ diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index bce2d59..f3cd47d 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -280,6 +280,7 @@ typedef enum NodeTag T_AllocSetContext, T_SlabContext, T_GenerationContext, + T_ShmRetailContext, /* * TAGS FOR VALUE NODES (value.h) diff --git a/src/include/utils/memutils.h b/src/include/utils/memutils.h index 106c83d..7dd2158 100644 --- a/src/include/utils/memutils.h +++ b/src/include/utils/memutils.h @@ -18,7 +18,7 @@ #define MEMUTILS_H #include "nodes/memnodes.h" - +#include "utils/dsa.h" /* * MaxAllocSize, MaxAllocHugeSize @@ -79,6 +79,8 @@ extern void MemoryContextDeleteChildren(MemoryContext context); extern void MemoryContextSetIdentifier(MemoryContext context, const char *id); extern void MemoryContextSetParent(MemoryContext context, MemoryContext new_parent); +extern MemoryContext MemoryContextClone(MemoryContext template, + MemoryContext parent); extern Size GetMemoryChunkSpace(void *pointer); extern MemoryContext MemoryContextGetParent(MemoryContext context); extern bool MemoryContextIsEmpty(MemoryContext context); @@ -182,6 +184,20 @@ extern MemoryContext GenerationContextCreate(MemoryContext parent, const char *name, Size blockSize); +/* shm_mcxt.c */ +extern MemoryContext ShmRetailContextCreateGlobal(MemoryContext parent, + const char *name, + dsa_area *area, + void *base); +extern MemoryContext ShmRetailContextCreateLocal(MemoryContext parent, + const char *name, + MemoryContext global_context); + +extern void ShmRetailContextMoveChunk(MemoryContext local_context, + MemoryContext global_context); + +extern Size ShmRetailContextSize(void); + /* * Recommended default alloc parameters, suitable for "ordinary" contexts * that might hold quite a lot of data. diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index b2eaef3..a00a282 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -22,6 +22,7 @@ SUBDIRS = \ test_rbtree \ test_rls_hooks \ test_shm_mq \ + test_shm_retail \ unsafe_tests \ worker_spi diff --git a/src/test/modules/test_shm_retail/.gitignore b/src/test/modules/test_shm_retail/.gitignore new file mode 100644 index 0000000..ba2160b --- /dev/null +++ b/src/test/modules/test_shm_retail/.gitignore @@ -0,0 +1,3 @@ +# Generated subdirectories +/output_iso/ +/tmp_check_iso/ diff --git a/src/test/modules/test_shm_retail/Makefile b/src/test/modules/test_shm_retail/Makefile new file mode 100644 index 0000000..a933bc0 --- /dev/null +++ b/src/test/modules/test_shm_retail/Makefile @@ -0,0 +1,31 @@ +# src/test/modules/test_shm_retail/Makefile + +MODULES = test_shm_retail +EXTENSION = test_shm_retail +DATA = test_shm_retail--1.0.sql +PGFILEDESC = "test_shm_retail - example use of shared memory context" + +ISOLATION = concurrent_test + +# enable our module in shared_preload_libraries +ISOLATION_OPTS = --temp-config $(top_srcdir)/src/test/modules/test_shm_retail/test_shm_retail.conf + +# Disabled because these tests require "shared_preload_libraries=test_shm_retail", +# which typical installcheck users do not have (e.g. buildfarm clients). +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_shm_retail +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +# But it can nonetheless be very helpful to run tests on preexisting +# installation, allow to do so, but only if requested explicitly. +installcheck-force: + $(pg_isolation_regress_installcheck) $(ISOLATION) diff --git a/src/test/modules/test_shm_retail/README b/src/test/modules/test_shm_retail/README new file mode 100644 index 0000000..d730857 --- /dev/null +++ b/src/test/modules/test_shm_retail/README @@ -0,0 +1,24 @@ +test_shm_retail is an example of how to use ShmContext and its facility. It +is not intended to do anything useful on its own; rather, it is a +demonstration of how these facilities can be used, and a unit test of +those facilities. + +This extension allows a backend to put a list of number into the shared +memory and another backend to get the list. To use this extension, you need +to add shared_preload_libraries = 'test_shm_retail'. + + +Functions +========= + +set_shared_list(i int) + RETURNS void + +This function sets positve integer into shared list and if a negative integer +is specified, delete the corresponding positive integer. + + +get_shared_list() + RETURNS SETOF integer + +This function retunrs intergers registerd to the shared list. \ No newline at end of file diff --git a/src/test/modules/test_shm_retail/expected/concurrent_test.out b/src/test/modules/test_shm_retail/expected/concurrent_test.out new file mode 100644 index 0000000..a17d880 --- /dev/null +++ b/src/test/modules/test_shm_retail/expected/concurrent_test.out @@ -0,0 +1,30 @@ +Parsed test spec with 2 sessions + +starting permutation: add_s1_1 add_s2_2 get_s1 remove_s1_2 remove_s2_1 get_s1 +step add_s1_1: CALL set_shared_list(1); +step add_s2_2: CALL set_shared_list(2); +step get_s1: SELECT * from get_shared_list(); +get_shared_list + +1 +2 +step remove_s1_2: CALL set_shared_list(-2); +step remove_s2_1: CALL set_shared_list(-1); +step get_s1: SELECT * from get_shared_list(); +get_shared_list + + +starting permutation: add_s1_1 get_s2 remove_s1_2 get_s1 add_s1_1 remove_s1_1 remove_s2_1 +step add_s1_1: CALL set_shared_list(1); +step get_s2: SELECT * from get_shared_list(); +get_shared_list + +1 +step remove_s1_2: CALL set_shared_list(-2); +step get_s1: SELECT * from get_shared_list(); +get_shared_list + +1 +step add_s1_1: CALL set_shared_list(1); +step remove_s1_1: CALL set_shared_list(-1); +step remove_s2_1: CALL set_shared_list(-1); diff --git a/src/test/modules/test_shm_retail/specs/concurrent_test.spec b/src/test/modules/test_shm_retail/specs/concurrent_test.spec new file mode 100644 index 0000000..0c2ac40 --- /dev/null +++ b/src/test/modules/test_shm_retail/specs/concurrent_test.spec @@ -0,0 +1,25 @@ +setup +{ + CREATE EXTENSION test_shm_retail; +} + +teardown +{ + DROP EXTENSION test_shm_retail; +} + +session "s1" +step "add_s1_1" {CALL set_shared_list(1);} +step "remove_s1_1" {CALL set_shared_list(-1);} +step "remove_s1_2" {CALL set_shared_list(-2);} +step "get_s1" {SELECT * from get_shared_list();} + +session "s2" +step "add_s2_2" {CALL set_shared_list(2);} +step "remove_s2_1" {CALL set_shared_list(-1);} +step "get_s2" {SELECT * from get_shared_list();} + +permutation "add_s1_1" "add_s2_2" "get_s1" "remove_s1_2" "remove_s2_1" "get_s1" +permutation "add_s1_1" "get_s2" "remove_s1_2" "get_s1" + +"add_s1_1" "remove_s1_1" "remove_s2_1" diff --git a/src/test/modules/test_shm_retail/test_shm_retail--1.0.sql b/src/test/modules/test_shm_retail/test_shm_retail--1.0.sql new file mode 100644 index 0000000..62a5087 --- /dev/null +++ b/src/test/modules/test_shm_retail/test_shm_retail--1.0.sql @@ -0,0 +1,14 @@ +/* src/test/modules/test_shm_retail/test_shm_retail--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_shm_retail" to load this file. \quit + + +CREATE PROCEDURE set_shared_list(i int) +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION get_shared_list() +RETURNS SETOF integer +AS 'MODULE_PATHNAME' +LANGUAGE C; diff --git a/src/test/modules/test_shm_retail/test_shm_retail.c b/src/test/modules/test_shm_retail/test_shm_retail.c new file mode 100644 index 0000000..6515de4 --- /dev/null +++ b/src/test/modules/test_shm_retail/test_shm_retail.c @@ -0,0 +1,197 @@ +/*-------------------------------------------------------------------------- + * + * test_shm_retail.c + * Code to test ShmRetailContext. + * + * This test code mimics global system catalog cache (catcache) flow. + * Global catcache has a hash table in shared memory and inserts or + * deletes a cache entry. Memory for a cache entry is allocated + * in shared memory as a chunk of local ShmRetailContext at first. + * The memory chunk of cache entry is moved to global ShmRetailContext + * after the cache is built and registered to hash table. This way prevents + * the chunk from leaked. + * + * Following this scenario, this test also creates index structure in + * shared memory and entries. The entry has the positive number provided + * by set_shared_list(). When the argument of set_shared_list() is + * positive it is registed to index and removed when it's negative. + * Too see the entries, use get_shared_list(). + * + * Copyright (c) 2013-2019, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_shm_retail/test_shm_retail.c + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "fmgr.h" +#include "funcapi.h" +#include "lib/ilist.h" +#include "miscadmin.h" +#include "nodes/pg_list.h" +#include "nodes/memnodes.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +#define MY_AREA_SIZE (1024 * 1024) + + +PG_MODULE_MAGIC; + +void _PG_init(void); +PG_FUNCTION_INFO_V1(set_shared_list); +PG_FUNCTION_INFO_V1(get_shared_list); + +static void shm_mcxt_shmem_startup_hook(void); + +static shmem_startup_hook_type prev_shmem_startup_hook; +static void *my_raw_memory; +static dsa_area *my_area; +static MemoryContext globalShmRetailContext; +static MemoryContext localShmRetailContext; + +static List **my_list; + + +void +_PG_init(void) +{ + /* This only works if preloaded by the postmaster. */ + if (!process_shared_preload_libraries_in_progress) + return; + + /* Request a chunk of traditional shared memory. */ + RequestAddinShmemSpace(MY_AREA_SIZE); + + /* Register our hook for phase II of initialization. */ + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = shm_mcxt_shmem_startup_hook; +} + +static void +shm_mcxt_shmem_startup_hook(void) +{ + MemoryContext old_context; + bool found; + + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + + old_context = MemoryContextSwitchTo(TopMemoryContext); + + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + + /* Allocate, or look up, a chunk of raw fixed-address shared memory. */ + my_raw_memory = ShmemInitStruct("my_area", MY_AREA_SIZE, &found); + if (!found) + { + /* + * Create a new DSA area, and clamp its size so it can't make any + * segments outside the provided space. + */ + my_area = dsa_create_in_place(my_raw_memory, MY_AREA_SIZE, 0, NULL); + dsa_set_size_limit(my_area, MY_AREA_SIZE); + } + else + { + /* Attach to an existing area. */ + my_area = dsa_attach_in_place(my_raw_memory, NULL); + } + + /* Also allocate or look up a list header. */ + my_list = ShmemInitStruct("my_index", MY_AREA_SIZE, &found); + if (!found) + *my_list = NIL; + + LWLockRelease(AddinShmemInitLock); + + /* Create a global memory context. */ + globalShmRetailContext = ShmRetailContextCreateGlobal(NULL, + "my_shared_context", + my_area, + my_raw_memory); + + MemoryContextSwitchTo(old_context); +} + +/* Set the positive number */ +Datum +set_shared_list(PG_FUNCTION_ARGS) +{ + int i = PG_GETARG_INT32(0); + MemoryContext old_context; + + /* Manipulate a list in shared memory. */ + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + if (i < 0) + *my_list = list_delete_int(*my_list, -i); + else + { + /* + * Create local context whose type is same as + * globalShmRetailContext and whose parent is CurrentMemoryContext. + */ + localShmRetailContext = + MemoryContextClone(globalShmRetailContext, + CurrentMemoryContext); + + old_context = MemoryContextSwitchTo(localShmRetailContext); + *my_list = lappend_int(*my_list, i); + + ShmRetailContextMoveChunk(localShmRetailContext, + globalShmRetailContext); + } + LWLockRelease(AddinShmemInitLock); + + MemoryContextSwitchTo(old_context); + + PG_RETURN_VOID(); +} + +/* Get the list of intergers registerd to shared list */ +Datum +get_shared_list(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + MemoryContext oldcontext; + int result; + ListCell **lcp; + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + + /* + * Switch to memory context appropriate for multiple function calls + */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + LWLockAcquire(AddinShmemInitLock, LW_SHARED); + /* allocate memory for user context to hold current cell */ + lcp = (ListCell **) palloc(sizeof(ListCell *)); + *lcp = list_head(*my_list); + funcctx->user_fctx = (void *) lcp; + + LWLockRelease(AddinShmemInitLock); + + MemoryContextSwitchTo(oldcontext); + } + + /* stuff done on every call of the function */ + funcctx = SRF_PERCALL_SETUP(); + lcp = (ListCell **) funcctx->user_fctx; + + while (*lcp != NULL) + { + result = lfirst_int(*lcp); + *lcp = lnext(*lcp); + SRF_RETURN_NEXT(funcctx, Int32GetDatum(result)); + } + + SRF_RETURN_DONE(funcctx); +} diff --git a/src/test/modules/test_shm_retail/test_shm_retail.conf b/src/test/modules/test_shm_retail/test_shm_retail.conf new file mode 100644 index 0000000..06f90a2 --- /dev/null +++ b/src/test/modules/test_shm_retail/test_shm_retail.conf @@ -0,0 +1 @@ +shared_preload_libraries = 'test_shm_retail' diff --git a/src/test/modules/test_shm_retail/test_shm_retail.control b/src/test/modules/test_shm_retail/test_shm_retail.control new file mode 100644 index 0000000..c6b7c60 --- /dev/null +++ b/src/test/modules/test_shm_retail/test_shm_retail.control @@ -0,0 +1,4 @@ +comment = 'Test code for shared retail memory context' +default_version = '1.0' +module_pathname = '$libdir/test_shm_retail' +relocatable = true \ No newline at end of file