commit 3f6c24f07577f2044091ded401f618d7d64e614e Author: Ashutosh Bapat Date: Tue Feb 17 16:51:20 2026 +0530 WIP: resizable shared memory structures diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index 2ebec6928d5..31cd2dabb54 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -4243,8 +4243,39 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx Size of the allocation in bytes including padding. For anonymous allocations, no information about padding is available, so the size and allocated_size columns - will always be equal. Padding is not meaningful for free memory, so - the columns will be equal in that case also. + will always be equal. Padding is not meaningful for free memory, so the + columns will be equal in that case also. For resizable allocations which + may span multiple memory pages, the padding includes the padding due to + page alignment. + + + + + + maximum_size int8 + + + Maximum size in bytes that the allocation can grow upto. For fixed-size allocations + allocated_size and + maxium_size are same. For anonymous + allocations, no information about maximum size is available, so the + size and maximum_size columns will + always be equal. Maximum size is not meaningful for free memory, so the + columns will be equal in that case also. + + + + + + allocated_space int8 + + + Address space, as against the memory, allocated for this allocation in + terms of bytes for resizable structures. It is greater than or equal to + allocated_size for these structures. It also + includes padding, if any. For fixed-size allocations, anonymous + allocations, and free memory this is same as + allocated_size. diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c index 2e3886cf9fe..25aef4173a9 100644 --- a/src/backend/port/sysv_shmem.c +++ b/src/backend/port/sysv_shmem.c @@ -589,6 +589,27 @@ check_huge_page_size(int *newval, void **extra, GucSource source) return true; } +/* + * Get the page size being used by the shared memory. + * + * The function should be called only after the shared memory has been setup. + */ +Size +GetOSPageSize(void) +{ + Size os_page_size; + + Assert(huge_pages_status != HUGE_PAGES_UNKNOWN); + + os_page_size = sysconf(_SC_PAGESIZE); + + /* If huge pages are actually in use, use huge page size */ + if (huge_pages_status == HUGE_PAGES_ON) + GetHugePageSize(&os_page_size, NULL); + + return os_page_size; +} + /* * Creates an anonymous mmap()ed shared memory segment. * @@ -991,3 +1012,51 @@ PGSharedMemoryDetach(void) AnonymousShmem = NULL; } } + +/* + * Make sure that the memory of given size from the given address is released. + * + * The address and size are expected to be page aligned. + * + * Only supported on platforms that support anonymous shared memory. + */ +void +PGSharedMemoryEnsureFreed(void *addr, Size size) +{ + if (!AnonymousShmem) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only anonymous shared memory can be freed"))); + + Assert(addr == (void *) TYPEALIGN(GetOSPageSize(), addr)); + Assert(size == TYPEALIGN(GetOSPageSize(), size)); + Assert(size > 0); + + if (madvise(addr, size, MADV_REMOVE) == -1) + ereport(ERROR, + (errmsg("could not free shared memory: %m"))); +} + +/* + * Make sure that the memory of given size from the given address is allocated. + * + * The address and size are expected to be page aligned. + * + * Only supported on platforms that support anonymous shared memory. + */ +void +PGSharedMemoryEnsureAllocated(void *addr, Size size) +{ + if (!AnonymousShmem) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only anonymous shared memory can be allocated at runtime"))); + + Assert(addr == (void *) TYPEALIGN(GetOSPageSize(), addr)); + Assert(size == TYPEALIGN(GetOSPageSize(), size)); + Assert(size > 0); + + if (madvise(addr, size, MADV_POPULATE_WRITE) == -1) + ereport(ERROR, + (errmsg("could not allocate shared memory: %m"))); +} diff --git a/src/backend/port/win32_shmem.c b/src/backend/port/win32_shmem.c index 794e4fcb2ad..4b07db206f4 100644 --- a/src/backend/port/win32_shmem.c +++ b/src/backend/port/win32_shmem.c @@ -621,6 +621,32 @@ pgwin32_ReserveSharedMemoryRegion(HANDLE hChild) return true; } +/* + * Make sure that the memory of given size from the given address is freed. + * + * Not supported on Windows currently. + */ +void +PGSharedMemoryEnsureFreed(void *addr, Size size) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("freeing shared memory is not supported on windows"))); +} + +/* + * Make sure that the memory of given size from the given address is allocated. + * + * Not supported on Windows currently. + */ +void +PGSharedMemoryEnsureAllocated(void *addr, Size size) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("allocating shared memory is not supported on windows"))); +} + /* * This function is provided for consistency with sysv_shmem.c and does not * provide any useful information for Windows. To obtain the large page size, @@ -648,3 +674,26 @@ check_huge_page_size(int *newval, void **extra, GucSource source) } return true; } + +/* + * Get the page size used by the shared memory. + * + * The function should be called only after the shared memory has been setup. + */ +Size +GetOSPageSize(void) +{ + SYSTEM_INFO sysinfo; + Size os_page_size; + + Assert(huge_pages_status != HUGE_PAGES_UNKNOWN); + + GetSystemInfo(&sysinfo); + os_page_size = sysinfo.dwPageSize; + + /* If huge pages are actually in use, use huge page size */ + if (huge_pages_status == HUGE_PAGES_ON) + GetHugePageSize(&os_page_size, NULL); + + return os_page_size; +} diff --git a/src/backend/storage/ipc/shmem.c b/src/backend/storage/ipc/shmem.c index d3808432ff1..8008f95ec44 100644 --- a/src/backend/storage/ipc/shmem.c +++ b/src/backend/storage/ipc/shmem.c @@ -171,6 +171,9 @@ typedef struct ShmemAreaKind kind; } ShmemRequest; +#define SHMEM_REQUEST_SPACE_SIZE(request) \ + ((request)->options->maximum_size > 0 ? (request)->options->maximum_size : (request)->options->size) + static List *requested_shmem_areas; /* @@ -270,12 +273,15 @@ typedef struct void *location; /* location in shared mem */ Size size; /* # bytes requested for the structure */ Size allocated_size; /* # bytes actually allocated */ + Size maximum_size; /* the maximum size a structure can grow to */ + Size allocated_space; /* the total address space allocated */ } ShmemIndexEnt; /* To get reliable results for NUMA inquiry we need to "touch pages" once */ static bool firstNumaTouch = true; static bool AttachOrInit(ShmemRequest *request, bool init_allowed, bool attach_allowed); +static Size EstimateAllocatedSize(ShmemIndexEnt *entry); Datum pg_numa_available(PG_FUNCTION_ARGS); @@ -345,16 +351,25 @@ ShmemRequestInternal(ShmemStructDesc *desc, ShmemRequestStructOpts *options, if (options->size <= 0 && options->size != SHMEM_REQUEST_UNKNOWN_SIZE) elog(ERROR, "invalid size %zd for shared memory request for \"%s\"", options->size, options->name); + if (options->maximum_size < 0 && options->maximum_size != SHMEM_REQUEST_UNKNOWN_SIZE) + elog(ERROR, "invalid maximum_size %zd for shared memory request for \"%s\"", + options->maximum_size, options->name); } else { if (options->size == SHMEM_REQUEST_UNKNOWN_SIZE) elog(ERROR, "SHMEM_REQUEST_UNKNOWN_SIZE cannot be used during startup"); - if (options->size <= 0) - elog(ERROR, "invalid size %zd for shared memory request for \"%s\"", - options->size, options->name); + if (options->maximum_size == SHMEM_REQUEST_UNKNOWN_SIZE) + elog(ERROR, "SHMEM_REQUEST_UNKNOWN_SIZE cannot be used during startup"); + if (options->maximum_size < 0) + elog(ERROR, "invalid maximum_size %zd for shared memory request for \"%s\"", + options->maximum_size, options->name); } + if (options->maximum_size > 0 && options->size >= options->maximum_size) + elog(ERROR, "resizable shared memory structure \"%s\" should have maximum size (%zd) greater than size (%zd)", + options->name, options->maximum_size, options->size); + if (shmem_startup_state != SB_REQUESTING) elog(ERROR, "ShmemRequestStruct can only be called from a shmem_request callback"); @@ -377,8 +392,12 @@ ShmemRequestInternal(ShmemStructDesc *desc, ShmemRequestStructOpts *options, } /* - * ShmemGetRequestedSize() --- estimate the total size of all registered shared - * memory structures. + * ShmemGetRequestedSize() --- estimate the total size of all registered shared + * memory structures. + * + * When maximum_size is specified for a request, we use that instead of the + * initial size for the estimation, to ensure that enough memory is reserved for + * resizable structures. * * This is called once at postmaster startup, before the shared memory segment * has been created. @@ -398,7 +417,7 @@ ShmemGetRequestedSize(void) { ShmemRequest *request = (ShmemRequest *) lfirst(lc); - size = add_size(size, request->options->size); + size = add_size(size, SHMEM_REQUEST_SPACE_SIZE(request)); size = add_size(size, request->options->extra_size); size = add_size(size, request->options->alignment); } @@ -566,13 +585,17 @@ AttachOrInit(ShmemRequest *request, bool init_allowed, bool attach_allowed) { /* * We inserted the entry to the shared memory index. Allocate - * requested amount of shared memory for it, and do basic - * initializion. + * requested amount of address space in the shared memory segment for it, and do basic + * initializion. The memory gets mapped during initialization as + * the corresponding memory pages are written to. Allocate enough space + * for a resizable structure to grow to its maximum size. It is expected + * that the initialization callback will use only as much + * memory as the initial size of the resizable structure. */ size_t allocated_size; void *structPtr; - structPtr = ShmemAllocRaw(request->options->size, request->options->alignment, &allocated_size); + structPtr = ShmemAllocRaw(SHMEM_REQUEST_SPACE_SIZE(request), request->options->alignment, &allocated_size); if (structPtr == NULL) { /* out of memory; remove the failed ShmemIndex entry */ @@ -584,11 +607,23 @@ AttachOrInit(ShmemRequest *request, bool init_allowed, bool attach_allowed) desc->name, request->options->size))); } index_entry->size = request->options->size; - index_entry->allocated_size = allocated_size; + index_entry->maximum_size = SHMEM_REQUEST_SPACE_SIZE(request); + index_entry->allocated_space = allocated_size; + if (request->options->maximum_size > 0) + { + /* Resizable structure. */ + index_entry->allocated_size = EstimateAllocatedSize(index_entry); + } + else + { + /* Fixed-size structure.*/ + index_entry->allocated_size = allocated_size; + } index_entry->location = structPtr; desc->ptr = index_entry->location; desc->size = index_entry->size; + desc->maximum_size = index_entry->maximum_size; switch (request->kind) { case SHMEM_KIND_STRUCT: @@ -607,6 +642,98 @@ AttachOrInit(ShmemRequest *request, bool init_allowed, bool attach_allowed) return found; } +/* + * Estimate the actual memory allocated for a resizable structure. + */ +static Size +EstimateAllocatedSize(ShmemIndexEnt *entry) +{ + Size page_size = GetOSPageSize(); + char *align_end = (char *) TYPEALIGN(page_size, (char *) entry->location + entry->size); + char *floor_max_end = (char *) TYPEALIGN_DOWN(page_size, (char *) entry->location + entry->maximum_size); + + Assert(entry->maximum_size >= entry->size); + Assert(entry->allocated_space >= entry->maximum_size); + + if (align_end >= floor_max_end) + { + /* + * A resizable structure which ends on the same page irrespective of its + * size. The structure will be allocated maximum memory at the beginning. + */ + return entry->allocated_space; + } + else + { + /* + * The maximal structure spans multiple pages. At the beginning the pages + * between the page where this structure, with its initial size, ends and + * the page where the next structure starts will not be allocated. + */ + return entry->allocated_space - (floor_max_end - align_end); + } +} + +void +ShmemResizeRegistered(const ShmemStructDesc *desc, Size new_size) +{ + ShmemIndexEnt *result; + bool found; + Size page_size = GetOSPageSize(); + char *new_end; + + Assert(new_size > 0); + + if (desc->maximum_size <= 0) + elog(ERROR, "shared memory struct \"%s\" is not resizable", desc->name); + + /* look it up in the shmem index */ + LWLockAcquire(ShmemIndexLock, LW_EXCLUSIVE); + result = (ShmemIndexEnt *) hash_search(ShmemIndex, desc->name, HASH_FIND, &found); + if (!found) + elog(ERROR, "shmem struct \"%s\" is not initialized", desc->name); + + Assert(result); + + if (result->maximum_size != desc->maximum_size) + elog(ERROR, "shmem struct \"%s\" has corrupted descriptor", desc->name); + + if (result->maximum_size < new_size) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("not enough address space is reserved for resizing structure \"%s\"", desc->name))); + + /* + * When shrinking the memory from the page aligned new end to the start of + * the page containing end of the reserved space is not required. Whereas + * when expanding the memory from the start of the page containing the start + * of the structure to the page aligned new end is required. + */ + new_end = (char *) TYPEALIGN(page_size, (char *) result->location + new_size); + if (new_size < result->size) + { + char *max_end = (char *) TYPEALIGN_DOWN(page_size, (char *) result->location + result->maximum_size); + Size free_size = max_end - new_end; + + if (free_size > 0) + PGSharedMemoryEnsureFreed(new_end, free_size); + } + else if (new_size > result->size) + { + char *struct_start = (char *) TYPEALIGN_DOWN(page_size, (char *) result->location); + Size alloc_size = new_end - struct_start; + + if (alloc_size > 0) + PGSharedMemoryEnsureAllocated(struct_start, alloc_size); + } + + /* Update shmem index entry. */ + result->size = new_size; + result->allocated_size = EstimateAllocatedSize(result); + + LWLockRelease(ShmemIndexLock); +} + /* * InitShmemAllocator() --- set up basic pointers to shared memory. * @@ -975,7 +1102,7 @@ mul_size(Size s1, Size s2) Datum pg_get_shmem_allocations(PG_FUNCTION_ARGS) { -#define PG_GET_SHMEM_SIZES_COLS 4 +#define PG_GET_SHMEM_SIZES_COLS 6 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; HASH_SEQ_STATUS hstat; ShmemIndexEnt *ent; @@ -997,7 +1124,15 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS) values[1] = Int64GetDatum((char *) ent->location - (char *) ShmemSegHdr); values[2] = Int64GetDatum(ent->size); values[3] = Int64GetDatum(ent->allocated_size); - named_allocated += ent->allocated_size; + values[4] = Int64GetDatum(ent->maximum_size); + values[5] = Int64GetDatum(ent->allocated_space); + + /* + * Keep track of the total allocated space for named shmem areas, to be + * able to calculate the amount of shared memory allocated for anonymous + * areas and the amount of free shared memory at the end of the segment. + */ + named_allocated += ent->allocated_space; tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -1008,6 +1143,8 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS) nulls[1] = true; values[2] = Int64GetDatum(ShmemAllocator->free_offset - named_allocated); values[3] = values[2]; + values[4] = values[2]; + values[5] = values[2]; tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); /* output as-of-yet unused shared memory */ @@ -1016,6 +1153,8 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS) nulls[1] = false; values[2] = Int64GetDatum(ShmemSegHdr->totalsize - ShmemAllocator->free_offset); values[3] = values[2]; + values[4] = values[2]; + values[5] = values[2]; tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); LWLockRelease(ShmemIndexLock); @@ -1203,23 +1342,9 @@ pg_get_shmem_allocations_numa(PG_FUNCTION_ARGS) Size pg_get_shmem_pagesize(void) { - Size os_page_size; -#ifdef WIN32 - SYSTEM_INFO sysinfo; - - GetSystemInfo(&sysinfo); - os_page_size = sysinfo.dwPageSize; -#else - os_page_size = sysconf(_SC_PAGESIZE); -#endif - Assert(IsUnderPostmaster); - Assert(huge_pages_status != HUGE_PAGES_UNKNOWN); - - if (huge_pages_status == HUGE_PAGES_ON) - GetHugePageSize(&os_page_size, NULL); - return os_page_size; + return GetOSPageSize(); } Datum diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0118e970dda..d4a4c3c5afd 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -8664,8 +8664,8 @@ { oid => '5052', descr => 'allocations from the main shared memory segment', proname => 'pg_get_shmem_allocations', prorows => '50', proretset => 't', provolatile => 'v', prorettype => 'record', proargtypes => '', - proallargtypes => '{text,int8,int8,int8}', proargmodes => '{o,o,o,o}', - proargnames => '{name,off,size,allocated_size}', + proallargtypes => '{text,int8,int8,int8,int8,int8}', proargmodes => '{o,o,o,o,o,o}', + proargnames => '{name,off,size,allocated_size,maximum_size,allocated_space}', prosrc => 'pg_get_shmem_allocations', proacl => '{POSTGRES=X,pg_read_all_stats=X}' }, diff --git a/src/include/storage/pg_shmem.h b/src/include/storage/pg_shmem.h index 10c7b065861..f0efbf2aec1 100644 --- a/src/include/storage/pg_shmem.h +++ b/src/include/storage/pg_shmem.h @@ -89,6 +89,9 @@ extern PGShmemHeader *PGSharedMemoryCreate(Size size, PGShmemHeader **shim); extern bool PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2); extern void PGSharedMemoryDetach(void); +extern void PGSharedMemoryEnsureFreed(void *addr, Size size); +extern void PGSharedMemoryEnsureAllocated(void *addr, Size size); extern void GetHugePageSize(Size *hugepagesize, int *mmap_flags); +extern Size GetOSPageSize(void); #endif /* PG_SHMEM_H */ diff --git a/src/include/storage/shmem.h b/src/include/storage/shmem.h index 150c86d5884..1a9f9e3b7db 100644 --- a/src/include/storage/shmem.h +++ b/src/include/storage/shmem.h @@ -37,6 +37,12 @@ typedef enum * * 'name' and 'size' are required. Initialize any optional fields that you * don't use to zeros. + * + * 'maximum_size' is the maximum size this resizable struct can grow to in future. + * For fixed-size structures, set it to 0. The memory for the maximum size is + * not allocated right away but the corresponding address space is reserved so + * that memory can be mapped to it when the structure grows or taken away from + * it when the structure shrinks. * * After registration, the shmem machinery reserves memory for the area, sets * '*ptr' to point to the allocation, and calls the callbacks at the right @@ -49,6 +55,7 @@ typedef struct ShmemStructDesc void *ptr; size_t size; + size_t maximum_size; } ShmemStructDesc; #define SHMEM_REQUEST_UNKNOWN_SIZE (-1) @@ -72,6 +79,14 @@ typedef struct ShmemRequestStructOpts */ size_t extra_size; + /* + * Maximum size this structure can grow upto in future. The memory is not + * allocated right away but the corresponding address space is reserved so + * that memory can be mapped to it when the structure grows. Typically + * should be used for resizable structures which need contiguous memory. + */ + size_t maximum_size; + /* * When the shmem area is initialized or attached to, pointer to it is * stored in *ptr. It usually points to a global variable, used to access @@ -214,6 +229,7 @@ extern void *ShmemAllocNoError(Size size); extern bool ShmemAddrIsValid(const void *addr); extern void RegisterShmemCallbacks(const ShmemCallbacks *callbacks); +extern void ShmemResizeRegistered(const ShmemStructDesc *desc, Size new_size); extern void ShmemRequestInternal(ShmemStructDesc *desc, ShmemRequestStructOpts *options, ShmemAreaKind kind); diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 62fab9f3c2f..3ef8228851a 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -14,6 +14,7 @@ SUBDIRS = \ libpq_pipeline \ oauth_validator \ plsample \ + resizable_shmem \ spgist_name_ops \ test_aio \ test_binaryheap \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 6799ba11e11..5244b4bdb8f 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -13,6 +13,7 @@ subdir('libpq_pipeline') subdir('nbtree') subdir('oauth_validator') subdir('plsample') +subdir('resizable_shmem') subdir('spgist_name_ops') subdir('ssl_passphrase_callback') subdir('test_aio') diff --git a/src/test/modules/resizable_shmem/Makefile b/src/test/modules/resizable_shmem/Makefile new file mode 100644 index 00000000000..f3bd8ac0c7f --- /dev/null +++ b/src/test/modules/resizable_shmem/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/resizable_shmem/Makefile + +MODULES = resizable_shmem +TAP_TESTS = 1 + +EXTENSION = resizable_shmem +DATA = resizable_shmem--1.0.sql +PGFILEDESC = "resizable_shmem - test module for resizable shared memory" + +# This test requires library to be loaded at the server start, so disable +# installcheck +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/resizable_shmem +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/src/makefiles/pgxs.mk +endif diff --git a/src/test/modules/resizable_shmem/meson.build b/src/test/modules/resizable_shmem/meson.build new file mode 100644 index 00000000000..493bbbc95c3 --- /dev/null +++ b/src/test/modules/resizable_shmem/meson.build @@ -0,0 +1,36 @@ +# src/test/modules/resizable_shmem/meson.build + +resizable_shmem_sources = files( + 'resizable_shmem.c', +) + +if host_system == 'windows' + resizable_shmem_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'resizable_shmem', + '--FILEDESC', 'resizable_shmem - test module for resizable shared memory',]) +endif + +resizable_shmem = shared_module('resizable_shmem', + resizable_shmem_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += resizable_shmem + +test_install_data += files( + 'resizable_shmem.control', + 'resizable_shmem--1.0.sql', +) + +tests += { + 'name': 'resizable_shmem', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'tap': { + 'tests': [ + 't/001_resizable_shmem.pl', + ], + # This test requires library to be loaded at the server start, so disable + # installcheck + 'runningcheck': false, + }, +} diff --git a/src/test/modules/resizable_shmem/resizable_shmem--1.0.sql b/src/test/modules/resizable_shmem/resizable_shmem--1.0.sql new file mode 100644 index 00000000000..c1bcb6117b6 --- /dev/null +++ b/src/test/modules/resizable_shmem/resizable_shmem--1.0.sql @@ -0,0 +1,37 @@ +/* src/test/modules/resizable_shmem/resizable_shmem--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION resizable_shmem" to load this file. \quit + +-- Function to resize the test structure in the shared memory +CREATE FUNCTION resizable_shmem_resize(new_entries integer) +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT; + +-- Function to write data to all entries in the test structure in shared memory +-- Writing all the entries makes sure that the memory is actually allocated and +-- mapped to the process, so that we can later measure the memory usage. +CREATE FUNCTION resizable_shmem_write(entry_value integer) +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT; + +-- Function to verify that specified number of initial entries have expected value. +-- Reading all the entries makes sure that the memory is actually mapped to the +-- process, so that we can later measure the memory usage. +CREATE FUNCTION resizable_shmem_read(entry_count integer, entry_value integer) +RETURNS boolean +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT; + +-- Function to report memory usage statistics of the calling backend +CREATE FUNCTION resizable_shmem_usage(OUT rss_anon bigint, OUT rss_file bigint, OUT rss_shmem bigint, OUT vm_size bigint) +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT; + +-- Function to get the shared memory page size +CREATE FUNCTION resizable_shmem_pagesize() +RETURNS integer +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT; diff --git a/src/test/modules/resizable_shmem/resizable_shmem.c b/src/test/modules/resizable_shmem/resizable_shmem.c new file mode 100644 index 00000000000..4c0bb43fc23 --- /dev/null +++ b/src/test/modules/resizable_shmem/resizable_shmem.c @@ -0,0 +1,279 @@ +/* ------------------------------------------------------------------------- + * + * resizable_shmem.c + * Test module for PostgreSQL's resizable shared memory functionality + * + * This module demonstrates and tests the resizable shared memory API + * provided by shmem.c/shmem.h. + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "storage/shmem.h" +#include "storage/spin.h" +#include "utils/builtins.h" +#include "utils/guc.h" +#include "utils/memutils.h" +#include "utils/timestamp.h" +#include "access/htup_details.h" + +#include + +PG_MODULE_MAGIC; + +/* + * Default amount of shared buffers and hence the amount of shared memory + * allocated by default is in hundreds of MBs. The memory allocated to the test + * structure will be noticeable only when it's in the same order. + */ +#define TEST_INITIAL_ENTRIES (25 * 1024 * 1024) /* Initial number of entries (100MB) */ +#define TEST_MAX_ENTRIES (100 * 1024 * 1024) /* Maximum number of entries (400MB, 4x initial) */ +#define TEST_ENTRY_SIZE sizeof(int32) /* Size of each entry */ + +/* + * Resizable test data structure stored in shared memory. + * + * We do not use any locks. The test performs resizing, reads and writes none of + * which are concurrent to keep the code and the test simple. + */ +typedef struct TestResizableShmemStruct +{ + /* Metadata */ + int32 num_entries; /* Number of entries that can fit */ + + /* Data area - variable size */ + int32 data[FLEXIBLE_ARRAY_MEMBER]; +} TestResizableShmemStruct; + +static ShmemStructDesc testShmemDesc; + +/* Global pointer to our shared memory structure */ +static TestResizableShmemStruct *resizable_shmem = NULL; + +static void resizable_shmem_request(void *arg); +static void resizable_shmem_shmem_init(void *arg); + +static const ShmemCallbacks pgss_shmem_callbacks = { + .request_fn = resizable_shmem_request, + .init_fn = resizable_shmem_shmem_init, +}; + +/* SQL-callable functions */ +PG_FUNCTION_INFO_V1(resizable_shmem_resize); +PG_FUNCTION_INFO_V1(resizable_shmem_write); +PG_FUNCTION_INFO_V1(resizable_shmem_read); +PG_FUNCTION_INFO_V1(resizable_shmem_usage); +PG_FUNCTION_INFO_V1(resizable_shmem_pagesize); + +/* + * Module load callback + */ +void +_PG_init(void) +{ + /* + * The module needs to be loaded via shared_preload_libraries to register + * shared memory structure. But if that's not the case, don't throw an error. + * The SQL functions check for existence of the shared memory data structure. + */ + if (!process_shared_preload_libraries_in_progress) + return; + +#ifdef EXEC_BACKEND + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("resizable_shmem is not supported in EXEC_BACKEND builds"))); +#endif + + RegisterShmemCallbacks(&pgss_shmem_callbacks); +} + +/* + * Request shared memory resources + */ +static void +resizable_shmem_request(void *arg) +{ + /* Register our resizable shared memory structure */ + ShmemRequestStruct(&testShmemDesc, &(ShmemRequestStructOpts) { + .name = "resizable_shmem", + .size = offsetof(TestResizableShmemStruct, data) + (TEST_INITIAL_ENTRIES * TEST_ENTRY_SIZE), + .maximum_size = offsetof(TestResizableShmemStruct, data) + (TEST_MAX_ENTRIES * TEST_ENTRY_SIZE), + .ptr = (void **) &resizable_shmem, + }); +} + +/* + * Initialize shared memory structure + */ +static void +resizable_shmem_shmem_init(void *arg) +{ + /* + * Shared memory structure should have been allocated with the requested + * size. Initialize the metadata. + */ + Assert(resizable_shmem != NULL); + Assert(testShmemDesc.size >= offsetof(TestResizableShmemStruct, data) + (TEST_INITIAL_ENTRIES * TEST_ENTRY_SIZE)); + Assert(testShmemDesc.maximum_size >= offsetof(TestResizableShmemStruct, data) + (TEST_MAX_ENTRIES * TEST_ENTRY_SIZE)); + + resizable_shmem->num_entries = TEST_INITIAL_ENTRIES; + memset(resizable_shmem->data, 0, TEST_INITIAL_ENTRIES * TEST_ENTRY_SIZE); +} + +/* + * Resize the shared memory structure to accommodate the specified number of + * entries. + */ +Datum +resizable_shmem_resize(PG_FUNCTION_ARGS) +{ +#ifndef EXEC_BACKEND + int32 new_entries = PG_GETARG_INT32(0); + Size new_size; + + if (!resizable_shmem) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("resizable_shmem is not initialized"))); + + new_size = offsetof(TestResizableShmemStruct, data) + (new_entries * TEST_ENTRY_SIZE); + ShmemResizeRegistered(&testShmemDesc, new_size); + resizable_shmem->num_entries = new_entries; + + PG_RETURN_VOID(); +#else + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("resizing shared memory is not supported in EXEC_BACKEND builds"))); +#endif +} + +/* + * Write the given integer value to all entries in the data array. + */ +Datum +resizable_shmem_write(PG_FUNCTION_ARGS) +{ + int32 entry_value = PG_GETARG_INT32(0); + int32 i; + + if (!resizable_shmem) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("resizable_shmem is not initialized"))); + + /* Write the value to all current entries */ + for (i = 0; i < resizable_shmem->num_entries; i++) + resizable_shmem->data[i] = entry_value; + + PG_RETURN_VOID(); +} + +/* + * Check whether the first 'entry_count' entries all have the expected 'entry_value'. + * Returns true if all match, false otherwise. + */ +Datum +resizable_shmem_read(PG_FUNCTION_ARGS) +{ + int32 entry_count = PG_GETARG_INT32(0); + int32 entry_value = PG_GETARG_INT32(1); + int32 i; + + if (resizable_shmem == NULL) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("resizable_shmem is not initialized"))); + + /* Validate entry_count */ + if (entry_count < 0 || entry_count > resizable_shmem->num_entries) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("entry_count %d is out of range (0..%d)", entry_count, resizable_shmem->num_entries))); + + /* Check if first entry_count entries have the expected value */ + for (i = 0; i < entry_count; i++) + { + if (resizable_shmem->data[i] != entry_value) + PG_RETURN_BOOL(false); + } + + PG_RETURN_BOOL(true); +} + +/* + * Report multiple memory usage statistics of the calling backend process + * as reported by the kernel. + * Returns RssAnon, RssFile, RssShmem, VmSize from /proc/self/status as a record. + * + * TODO: See TODO note in SQL definition of this function. + */ +Datum +resizable_shmem_usage(PG_FUNCTION_ARGS) +{ + FILE *f; + char line[256]; + int64 rss_anon_kb = -1; + int64 rss_file_kb = -1; + int64 rss_shmem_kb = -1; + int64 vm_size_kb = -1; + int found = 0; + TupleDesc tupdesc; + Datum values[4]; + bool nulls[4]; + HeapTuple tuple; + + /* Open /proc/self/status to read memory information */ + f = fopen("/proc/self/status", "r"); + if (f == NULL) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open /proc/self/status: %m"))); + + /* Look for the memory usage lines */ + while (fgets(line, sizeof(line), f) != NULL && found < 4) + { + if (rss_anon_kb == -1 && sscanf(line, "RssAnon: %ld kB", &rss_anon_kb) == 1) + found++; + else if (rss_file_kb == -1 && sscanf(line, "RssFile: %ld kB", &rss_file_kb) == 1) + found++; + else if (rss_shmem_kb == -1 && sscanf(line, "RssShmem: %ld kB", &rss_shmem_kb) == 1) + found++; + else if (vm_size_kb == -1 && sscanf(line, "VmSize: %ld kB", &vm_size_kb) == 1) + found++; + } + + fclose(f); + + /* Build tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("function returning record called in context " + "that cannot accept a record"))); + + /* Build the result tuple */ + values[0] = Int64GetDatum(rss_anon_kb >= 0 ? rss_anon_kb * 1024 : 0); + values[1] = Int64GetDatum(rss_file_kb >= 0 ? rss_file_kb * 1024 : 0); + values[2] = Int64GetDatum(rss_shmem_kb >= 0 ? rss_shmem_kb * 1024 : 0); + values[3] = Int64GetDatum(vm_size_kb >= 0 ? vm_size_kb * 1024 : 0); + + nulls[0] = nulls[1] = nulls[2] = nulls[3] = false; + + tuple = heap_form_tuple(tupdesc, values, nulls); + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} + +/* + * resizable_shmem_pagesize() - Get the shared memory page size + */ +Datum +resizable_shmem_pagesize(PG_FUNCTION_ARGS) +{ + PG_RETURN_INT32(pg_get_shmem_pagesize()); +} diff --git a/src/test/modules/resizable_shmem/resizable_shmem.control b/src/test/modules/resizable_shmem/resizable_shmem.control new file mode 100644 index 00000000000..1ce2c5ea21a --- /dev/null +++ b/src/test/modules/resizable_shmem/resizable_shmem.control @@ -0,0 +1,5 @@ +# resizable_shmem extension test module +comment = 'test module for testing resizable shared memory structure functionality' +default_version = '1.0' +module_pathname = '$libdir/resizable_shmem' +relocatable = true diff --git a/src/test/modules/resizable_shmem/t/001_resizable_shmem.pl b/src/test/modules/resizable_shmem/t/001_resizable_shmem.pl new file mode 100644 index 00000000000..41a06c95bcd --- /dev/null +++ b/src/test/modules/resizable_shmem/t/001_resizable_shmem.pl @@ -0,0 +1,118 @@ +#!/usr/bin/perl +# Copyright (c) 2026, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test resizable shared memory functionality +# This converts the isolation test resizable_shmem.spec into a TAP test + +my $node = PostgreSQL::Test::Cluster->new('resizable_shmem'); + +# Need to configure for resizable_shmem +$node->init; +$node->append_conf('postgresql.conf', 'shared_preload_libraries = resizable_shmem'); +$node->start; + +# Create extension +$node->safe_psql('postgres', 'CREATE EXTENSION resizable_shmem;'); + +# Query string variables for reuse +my $rss_usage_query = 'SELECT rss_shmem FROM resizable_shmem_usage();'; +my $alloc_size_query = "SELECT allocated_size FROM pg_shmem_allocations WHERE name = 'resizable_shmem';"; +# Currently only one structure is resizable +my $fixed_struct_query = "SELECT count(*) FROM pg_shmem_allocations WHERE name <> 'resizable_shmem' and size <> maximum_size;"; + +my $page_size = $node->safe_psql('postgres', "SELECT resizable_shmem_pagesize();"); + +# Create background sessions for testing +my $session1 = $node->background_psql('postgres'); +my $session2 = $node->background_psql('postgres'); + +my $num_entries = 25 * 1024 * 1024; # Initial number of entries in resizable shared memory +my $max_entries = 100 * 1024 * 1024; # Maximum number of entries allowed +my $entry_size = 4; # each entry is int32 +my $prev_shmem_usage1 = $session1->query_safe($rss_usage_query, verbose => 0); +my $prev_shmem_usage2 = $session2->query_safe($rss_usage_query, verbose => 0); +my $prev_alloc_size; + +# We need to make sure that the changes to shared memory allocated are +# proportionate to the changes in the resizable shared memory structure. But +# there is no way to know the shared memory allocated at the given address in a +# given process. We can only know the size of shared memory accessed by the a +# given process. In case of PostgreSQL, that includes the memory allocated to +# other shared memory structures as well. Instead, we just note the changes in +# the function below to help in debugging overallocation issues. +sub note_shmem_changes +{ + my ($prev_shmem_usage1, $prev_shmem_usage2, $prev_alloc_size) = @_; + + my $shmem_usage1 = $session1->query_safe($rss_usage_query, verbose => 0); + my $shmem_usage2 = $session2->query_safe($rss_usage_query, verbose => 0); + my $alloc_size = $node->safe_psql('postgres', $alloc_size_query, verbose => 0); + + note "changes in allocated size: " . ($alloc_size - $prev_alloc_size); + note "Session 1: changes in rss_shmem usage: " . ($shmem_usage1 - $prev_shmem_usage1); + note "Session 1: difference in rss_shmem change and allocated size change: " . (($shmem_usage1 - $prev_shmem_usage1) - ($alloc_size - $prev_alloc_size)); + note "Session 2: changes in rss_shmem usage: " . ($shmem_usage2 - $prev_shmem_usage2); + note "Session 2: difference in rss_shmem change and allocated size change: " . (($shmem_usage2 - $prev_shmem_usage2) - ($alloc_size - $prev_alloc_size)); + + return ($shmem_usage1, $shmem_usage2, $alloc_size); +} + +my $value = 100; +# Write and read the initial set of entries. +$session1->query_safe("SELECT resizable_shmem_write($value);", verbose => 0); +is($session2->query_safe("SELECT resizable_shmem_read($num_entries, $value);", verbose => 0), 't', 'data read after write successful'); +($prev_shmem_usage1, $prev_shmem_usage2, $prev_alloc_size) = note_shmem_changes($prev_shmem_usage1, $prev_shmem_usage2, 0); +is($node->safe_psql('postgres', $fixed_struct_query), '0', 'initial fixed sized structures'); + +# Resize to maximum +my $old_num_entries = $num_entries; +$num_entries = $max_entries; +$session1->query_safe("SELECT resizable_shmem_resize($num_entries);", verbose => 0); +# Old data after resize should still be intact +is($session1->query_safe("SELECT resizable_shmem_read($old_num_entries, $value);", verbose => 0), 't', 'initial data readable after resize'); +$value = 500; +$session2->query_safe("SELECT resizable_shmem_write($value);", verbose => 0); +is($session1->query_safe("SELECT resizable_shmem_read($num_entries, $value);", verbose => 0), 't', 'enlarged area data read successful'); +($prev_shmem_usage1, $prev_shmem_usage2, $prev_alloc_size) = note_shmem_changes($prev_shmem_usage1, $prev_shmem_usage2, $prev_alloc_size); +is($node->safe_psql('postgres', $fixed_struct_query), '0', 'fixed sized structures after resize to maximum'); + +# Shrink smaller size +$old_num_entries = $num_entries; +$num_entries = 75 * 1024 * 1024; +$session2->query_safe("SELECT resizable_shmem_resize($num_entries);", verbose => 0); +# Old values should remain intact in the shrunk area +is($session1->query_safe("SELECT resizable_shmem_read($num_entries, $value);", verbose => 0), 't', 'data readable after shrinking'); +$value = 999; +$session1->query_safe("SELECT resizable_shmem_write($value);", verbose => 0); +is($session2->query_safe("SELECT resizable_shmem_read($num_entries, $value);", verbose => 0), 't', 'new data readable in shrunken area'); +($prev_shmem_usage1, $prev_shmem_usage2, $prev_alloc_size) = note_shmem_changes($prev_shmem_usage1, $prev_shmem_usage2, $prev_alloc_size); +is($node->safe_psql('postgres', $fixed_struct_query), '0', 'fixed sized structures after shrinking'); + +# Resize to the same size +$session2->query_safe("SELECT resizable_shmem_resize($num_entries);", verbose => 0); +# Old values should remain intact in the shrunk area +is($session1->query_safe("SELECT resizable_shmem_read($num_entries, $value);", verbose => 0), 't', 'data readable after shrinking'); +$value = 1999; +$session1->query_safe("SELECT resizable_shmem_write($value);", verbose => 0); +is($session2->query_safe("SELECT resizable_shmem_read($num_entries, $value);", verbose => 0), 't', 'new data readable in shrunken area'); +($prev_shmem_usage1, $prev_shmem_usage2, $prev_alloc_size) = note_shmem_changes($prev_shmem_usage1, $prev_shmem_usage2, $prev_alloc_size); +is($node->safe_psql('postgres', $fixed_struct_query), '0', 'fixed sized structures at the end'); + +# Test resize failure (attempt to resize beyond max - should fail) +my ($ret, $stdout, $stderr) = $node->psql('postgres', "SELECT resizable_shmem_resize(" . ($max_entries * 2) . ");"); +ok($ret != 0 || $stderr =~ /ERROR/, 'Resize beyond maximum fails'); + +# Cleanup sessions +$session1->quit; +$session2->quit; + +# Cleanup +$node->stop; + +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 2b3cf6d8569..d08e1ffe59d 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1770,8 +1770,9 @@ pg_shadow| SELECT pg_authid.rolname AS usename, pg_shmem_allocations| SELECT name, off, size, - allocated_size - FROM pg_get_shmem_allocations() pg_get_shmem_allocations(name, off, size, allocated_size); + allocated_size, + reserved_size + FROM pg_get_shmem_allocations() pg_get_shmem_allocations(name, off, size, allocated_size, maximum_size); pg_shmem_allocations_numa| SELECT name, numa_node, size