From c1956491c3baed58f91d5ac461bc2fe53d3f8d35 Mon Sep 17 00:00:00 2001 From: Anthonin Bonnefoy Date: Tue, 20 Jan 2026 10:55:33 +0100 Subject: Add new auto-tune shared_buffers GUC With some environments, it's not possible to modify the amount of huge pages and only a fixed quantity is available. shared_buffers can be adjusted to try to fit and use as much available memory as possible. However, this is very brittle as modifying other parameters, like the amount of max_connections, will change the requested shared memory, and shared_buffers will need to be further adjusted. This patch introduces a new GUC to dynamically increase shared_buffers. This GUC specifies an amount of shared memory to target, and shared_buffers will automatically be increased to use the available extra space. --- doc/src/sgml/config.sgml | 18 ++++++++ src/backend/port/sysv_shmem.c | 10 ++--- src/backend/storage/buffer/buf_init.c | 44 +++++++++++++++++++ src/backend/utils/misc/guc_parameters.dat | 9 ++++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/storage/bufmgr.h | 2 + 6 files changed, 79 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 5a152ee0885..9097226be43 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1758,6 +1758,24 @@ include_dir 'conf.d' + + shared_buffers_autotune_target (integer) + + shared_buffers_autotune_target configuration parameter + + + + + Specifies the amount of shared memory to target. If the requested + shared memory is inferior to the shared memory target, + shared_buffers will be increased to absorb the + extra space. + + This parameter can only be set at server start. + + + + huge_pages (enum) diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c index 3cd3544fa2b..8cf4ca8ee76 100644 --- a/src/backend/port/sysv_shmem.c +++ b/src/backend/port/sysv_shmem.c @@ -465,11 +465,11 @@ PGSharedMemoryAttach(IpcMemoryId shmId, * size to avoid trouble. * * Doing the round-up ourselves also lets us make use of the extra memory, - * rather than just wasting it. Currently, we just increase the available - * space recorded in the shmem header, which will make the extra usable for - * purposes such as additional locktable entries. Someday, for very large - * hugepage sizes, we might want to think about more invasive strategies, - * such as increasing shared_buffers to absorb the extra space. + * rather than just wasting it. We increase the available space recorded in + * the shmem header, which will make the extra usable for purposes such as + * additional locktable entries. If buffer autotune is enabled, + * shared_buffers will be automatically increased to use available extra + * space. * * Returns the (real, assumed or config provided) page size into * *hugepagesize, and the hugepage-related mmap flags to use into diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c index 6a57ab9cf30..7995e99d212 100644 --- a/src/backend/storage/buffer/buf_init.c +++ b/src/backend/storage/buffer/buf_init.c @@ -14,6 +14,7 @@ */ #include "postgres.h" +#include "utils/guc.h" #include "storage/aio.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" @@ -21,6 +22,7 @@ BufferDescPadded *BufferDescriptors; char *BufferBlocks; +int NBuffersTarget = 0; ConditionVariableMinimallyPadded *BufferIOCVArray; WritebackContext BackendWritebackContext; CkptSortItem *CkptBufferIds; @@ -142,6 +144,48 @@ BufferManagerShmemInit(void) &backend_flush_after); } +/* + * BufferManagerAutotune + * + * auto-tune shared_buffers to use memory up to NBuffersTarget + */ +void +BufferManagerAutotune(Size requested_size) +{ + char buf[32]; + Size target_shared_memory; + Size leftover_memory; + Size size_per_buffer; + Size additional_freelist_memory; + int candidate_nbuffers; + + target_shared_memory = mul_size(NBuffersTarget, BLCKSZ); + if (target_shared_memory <= requested_size) + /* target below requested size, nothing to do */ + return; + + leftover_memory = target_shared_memory - requested_size; + size_per_buffer = sizeof(BufferDescPadded) + + sizeof(ConditionVariableMinimallyPadded) + + sizeof(CkptSortItem) + BLCKSZ; + candidate_nbuffers = add_size(NBuffers, leftover_memory / size_per_buffer); + + /* + * With the additional shared_buffers, the shared memory necessary for + * freelist-related structures will increase. We need to estimate this + * additional memory, and reduce the auto-tuned shared_buffers to fit. + */ + additional_freelist_memory = StrategyShmemSize(candidate_nbuffers) - StrategyShmemSize(NBuffers); + candidate_nbuffers -= additional_freelist_memory / size_per_buffer + 1; + + if (candidate_nbuffers <= NBuffers) + return; + + snprintf(buf, sizeof(buf), "%d", candidate_nbuffers); + SetConfigOption("shared_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); +} + /* * BufferManagerShmemSize * diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index 48590622b95..ba5fd26758b 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -2609,6 +2609,15 @@ max => 'INT_MAX / 2', }, +{ name => 'shared_buffers_autotune_target', type => 'int', context => 'PGC_POSTMASTER', group => 'RESOURCES_MEM', + short_desc => 'Sets the number of shared memory buffers to target.', + flags => 'GUC_UNIT_BLOCKS', + variable => 'NBuffersTarget', + boot_val => '0', + min => '0', + max => 'INT_MAX / 2', +}, + { name => 'shared_memory_size', type => 'int', context => 'PGC_INTERNAL', group => 'PRESET_OPTIONS', short_desc => 'Shows the size of the server\'s main shared memory area (rounded up to the nearest MB).', flags => 'GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE | GUC_UNIT_MB | GUC_RUNTIME_COMPUTED', diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 256e8040092..60780cec05d 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -131,6 +131,7 @@ #shared_buffers = 128MB # min 128kB # (change requires restart) +#shared_buffers_autotune_target = 0 # (change requires restart) #huge_pages = try # on, off, or try # (change requires restart) #huge_page_size = 0 # zero for system default diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index a40adf6b2a8..384ccd3fa87 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -185,6 +185,7 @@ extern PGDLLIMPORT const PgAioHandleCallbacks aio_local_buffer_readv_cb; /* in buf_init.c */ extern PGDLLIMPORT char *BufferBlocks; +extern PGDLLIMPORT int NBuffersTarget; /* in localbuf.c */ extern PGDLLIMPORT int NLocBuffer; @@ -368,6 +369,7 @@ extern void MarkDirtyAllUnpinnedBuffers(int32 *buffers_dirtied, /* in buf_init.c */ extern void BufferManagerShmemInit(void); extern Size BufferManagerShmemSize(void); +extern void BufferManagerAutotune(Size requested_size); /* in localbuf.c */ extern void AtProcExit_LocalBuffers(void); -- 2.52.0