From c3a348938ad6eb3abf7ce7f520f10fb8c1404b14 Mon Sep 17 00:00:00 2001 From: Anthonin Bonnefoy Date: Mon, 27 Oct 2025 09:27:06 +0100 Subject: Create dedicated shmem Autotune functions Some subsystems auto-tune their shmem requests based on the shared_buffers value. The current behaviour is inconsistent: - Aio and XLOG modify their respective GUCs in *ShmemSize - CLOG, CommitTs, SUBTRANS modify their respective GUCs in *ShmemInit - Checkpointer doesn't save the auto-tuned value This patch introduces dedicated auto-tune functions for Aio, XLOG, CLOG, CommitTS and SUBTRANS. A new AutotuneShmem function is responsible for calling all the new auto-tune functions. --- src/backend/access/transam/clog.c | 52 ++++++++++++------------ src/backend/access/transam/commit_ts.c | 52 ++++++++++++------------ src/backend/access/transam/subtrans.c | 56 +++++++++++++------------- src/backend/access/transam/xlog.c | 46 ++++++++++++--------- src/backend/bootstrap/bootstrap.c | 5 +++ src/backend/postmaster/postmaster.c | 5 +++ src/backend/storage/aio/aio_init.c | 31 ++++++++------ src/backend/storage/ipc/ipci.c | 22 ++++++++++ src/backend/tcop/postgres.c | 5 +++ src/include/access/clog.h | 1 + src/include/access/commit_ts.h | 1 + src/include/access/subtrans.h | 1 + src/include/access/xlog.h | 1 + src/include/storage/aio_subsys.h | 1 + src/include/storage/ipc.h | 1 + 15 files changed, 170 insertions(+), 110 deletions(-) diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c index b5c38bbb162..2925ecbece4 100644 --- a/src/backend/access/transam/clog.c +++ b/src/backend/access/transam/clog.c @@ -756,6 +756,32 @@ TransactionIdGetStatus(TransactionId xid, XLogRecPtr *lsn) return status; } +/* + * Auto-tune transaction_buffers based on shared buffers + */ +void +CLOGAutotune(void) +{ + char buf[32]; + + if (transaction_buffers != 0) + return; + + snprintf(buf, sizeof(buf), "%d", SimpleLruAutotuneBuffers(512, 1024)); + SetConfigOption("transaction_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + + /* + * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. + * However, if the DBA explicitly set transaction_buffers = 0 in the + * config file, then PGC_S_DYNAMIC_DEFAULT will fail to override that and + * we must force the matter with PGC_S_OVERRIDE. + */ + if (transaction_buffers == 0) /* failed to apply it? */ + SetConfigOption("transaction_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); +} + /* * Number of shared CLOG buffers. * @@ -766,10 +792,7 @@ TransactionIdGetStatus(TransactionId xid, XLogRecPtr *lsn) static int CLOGShmemBuffers(void) { - /* auto-tune based on shared buffers */ - if (transaction_buffers == 0) - return SimpleLruAutotuneBuffers(512, 1024); - + Assert(transaction_buffers > 0); return Min(Max(16, transaction_buffers), CLOG_MAX_ALLOWED_BUFFERS); } @@ -785,27 +808,6 @@ CLOGShmemSize(void) void CLOGShmemInit(void) { - /* If auto-tuning is requested, now is the time to do it */ - if (transaction_buffers == 0) - { - char buf[32]; - - snprintf(buf, sizeof(buf), "%d", CLOGShmemBuffers()); - SetConfigOption("transaction_buffers", buf, PGC_POSTMASTER, - PGC_S_DYNAMIC_DEFAULT); - - /* - * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. - * However, if the DBA explicitly set transaction_buffers = 0 in the - * config file, then PGC_S_DYNAMIC_DEFAULT will fail to override that - * and we must force the matter with PGC_S_OVERRIDE. - */ - if (transaction_buffers == 0) /* failed to apply it? */ - SetConfigOption("transaction_buffers", buf, PGC_POSTMASTER, - PGC_S_OVERRIDE); - } - Assert(transaction_buffers != 0); - XactCtl->PagePrecedes = CLOGPagePrecedes; SimpleLruInit(XactCtl, "transaction", CLOGShmemBuffers(), CLOG_LSNS_PER_PAGE, "pg_xact", LWTRANCHE_XACT_BUFFER, diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c index 082b564da8f..f76b43bf25a 100644 --- a/src/backend/access/transam/commit_ts.c +++ b/src/backend/access/transam/commit_ts.c @@ -493,6 +493,32 @@ pg_xact_commit_timestamp_origin(PG_FUNCTION_ARGS) PG_RETURN_DATUM(HeapTupleGetDatum(htup)); } +/* + * Auto-tune commit_timestamp_buffers based on shared buffers + */ +void +CommitTsAutotune(void) +{ + char buf[32]; + + if (commit_timestamp_buffers != 0) + return; + + snprintf(buf, sizeof(buf), "%d", SimpleLruAutotuneBuffers(512, 1024)); + SetConfigOption("commit_timestamp_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + + /* + * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. + * However, if the DBA explicitly set commit_timestamp_buffers = 0 in the + * config file, then PGC_S_DYNAMIC_DEFAULT will fail to override that and + * we must force the matter with PGC_S_OVERRIDE. + */ + if (commit_timestamp_buffers == 0) /* failed to apply it? */ + SetConfigOption("commit_timestamp_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); +} + /* * Number of shared CommitTS buffers. * @@ -503,10 +529,7 @@ pg_xact_commit_timestamp_origin(PG_FUNCTION_ARGS) static int CommitTsShmemBuffers(void) { - /* auto-tune based on shared buffers */ - if (commit_timestamp_buffers == 0) - return SimpleLruAutotuneBuffers(512, 1024); - + Assert(commit_timestamp_buffers > 0); return Min(Max(16, commit_timestamp_buffers), SLRU_MAX_ALLOWED_BUFFERS); } @@ -529,27 +552,6 @@ CommitTsShmemInit(void) { bool found; - /* If auto-tuning is requested, now is the time to do it */ - if (commit_timestamp_buffers == 0) - { - char buf[32]; - - snprintf(buf, sizeof(buf), "%d", CommitTsShmemBuffers()); - SetConfigOption("commit_timestamp_buffers", buf, PGC_POSTMASTER, - PGC_S_DYNAMIC_DEFAULT); - - /* - * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. - * However, if the DBA explicitly set commit_timestamp_buffers = 0 in - * the config file, then PGC_S_DYNAMIC_DEFAULT will fail to override - * that and we must force the matter with PGC_S_OVERRIDE. - */ - if (commit_timestamp_buffers == 0) /* failed to apply it? */ - SetConfigOption("commit_timestamp_buffers", buf, PGC_POSTMASTER, - PGC_S_OVERRIDE); - } - Assert(commit_timestamp_buffers != 0); - CommitTsCtl->PagePrecedes = CommitTsPagePrecedes; SimpleLruInit(CommitTsCtl, "commit_timestamp", CommitTsShmemBuffers(), 0, "pg_commit_ts", LWTRANCHE_COMMITTS_BUFFER, diff --git a/src/backend/access/transam/subtrans.c b/src/backend/access/transam/subtrans.c index c0987f43f11..32433b0333f 100644 --- a/src/backend/access/transam/subtrans.c +++ b/src/backend/access/transam/subtrans.c @@ -191,18 +191,11 @@ SubTransGetTopmostTransaction(TransactionId xid) /* * Number of shared SUBTRANS buffers. - * - * If asked to autotune, use 2MB for every 1GB of shared buffers, up to 8MB. - * Otherwise just cap the configured amount to be between 16 and the maximum - * allowed. */ static int SUBTRANSShmemBuffers(void) { - /* auto-tune based on shared buffers */ - if (subtransaction_buffers == 0) - return SimpleLruAutotuneBuffers(512, 1024); - + Assert(subtransaction_buffers > 0); return Min(Max(16, subtransaction_buffers), SLRU_MAX_ALLOWED_BUFFERS); } @@ -215,30 +208,39 @@ SUBTRANSShmemSize(void) return SimpleLruShmemSize(SUBTRANSShmemBuffers(), 0); } +/* + * Auto-tune subtransaction_buffers based on shared_buffers + * + * If asked to autotune, use 2MB for every 1GB of shared buffers, up to 8MB. + * Otherwise just cap the configured amount to be between 16 and the maximum + * allowed. + */ void -SUBTRANSShmemInit(void) +SUBTRANSAutotune(void) { - /* If auto-tuning is requested, now is the time to do it */ - if (subtransaction_buffers == 0) - { - char buf[32]; + char buf[32]; - snprintf(buf, sizeof(buf), "%d", SUBTRANSShmemBuffers()); - SetConfigOption("subtransaction_buffers", buf, PGC_POSTMASTER, - PGC_S_DYNAMIC_DEFAULT); + if (subtransaction_buffers != 0) + return; - /* - * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. - * However, if the DBA explicitly set subtransaction_buffers = 0 in - * the config file, then PGC_S_DYNAMIC_DEFAULT will fail to override - * that and we must force the matter with PGC_S_OVERRIDE. - */ - if (subtransaction_buffers == 0) /* failed to apply it? */ - SetConfigOption("subtransaction_buffers", buf, PGC_POSTMASTER, - PGC_S_OVERRIDE); - } - Assert(subtransaction_buffers != 0); + snprintf(buf, sizeof(buf), "%d", SimpleLruAutotuneBuffers(512, 1024)); + SetConfigOption("subtransaction_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + /* + * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. + * However, if the DBA explicitly set subtransaction_buffers = 0 in the + * config file, then PGC_S_DYNAMIC_DEFAULT will fail to override that and + * we must force the matter with PGC_S_OVERRIDE. + */ + if (subtransaction_buffers == 0) /* failed to apply it? */ + SetConfigOption("subtransaction_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); +} + +void +SUBTRANSShmemInit(void) +{ SubTransCtl->PagePrecedes = SubTransPagePrecedes; SimpleLruInit(SubTransCtl, "subtransaction", SUBTRANSShmemBuffers(), 0, "pg_subtrans", LWTRANCHE_SUBTRANS_BUFFER, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 81dc86847c0..fee808dd8d7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4716,7 +4716,7 @@ check_wal_buffers(int *newval, void **extra, GucSource source) { /* * If we haven't yet changed the boot_val default of -1, just let it - * be. We'll fix it when XLOGShmemSize is called. + * be. We'll fix it when XLOGAutotune is called. */ if (XLOGbuffers == -1) return true; @@ -4956,35 +4956,41 @@ GetActiveWalLevelOnStandby(void) } /* - * Initialization of shared memory for XLOG + * Auto-tune wal_buffers value + * + * If the value of wal_buffers is -1, use the preferred auto-tune value. */ -Size -XLOGShmemSize(void) +void +XLOGAutotune(void) { - Size size; + char buf[32]; + + if (XLOGbuffers != -1) + return; + + snprintf(buf, sizeof(buf), "%d", XLOGChooseNumBuffers()); + SetConfigOption("wal_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); /* - * If the value of wal_buffers is -1, use the preferred auto-tune value. - * This isn't an amazingly clean place to do this, but we must wait till - * NBuffers has received its final value, and must do it before using the - * value of XLOGbuffers to do anything important. - * * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. * However, if the DBA explicitly set wal_buffers = -1 in the config file, * then PGC_S_DYNAMIC_DEFAULT will fail to override that and we must force * the matter with PGC_S_OVERRIDE. */ - if (XLOGbuffers == -1) - { - char buf[32]; - - snprintf(buf, sizeof(buf), "%d", XLOGChooseNumBuffers()); + if (XLOGbuffers == -1) /* failed to apply it? */ SetConfigOption("wal_buffers", buf, PGC_POSTMASTER, - PGC_S_DYNAMIC_DEFAULT); - if (XLOGbuffers == -1) /* failed to apply it? */ - SetConfigOption("wal_buffers", buf, PGC_POSTMASTER, - PGC_S_OVERRIDE); - } + PGC_S_OVERRIDE); +} + +/* + * Initialization of shared memory for XLOG + */ +Size +XLOGShmemSize(void) +{ + Size size; + Assert(XLOGbuffers > 0); /* XLogCtl */ diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index dd57624b4f9..4bbedb44b53 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -337,6 +337,11 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) InitializeFastPathLocks(); + /* + * Give the chance for subsystems to auto-tune their values. + */ + AutotuneShmem(); + CreateSharedMemoryAndSemaphores(); /* diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d6133bfebc6..f4e60274924 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -963,6 +963,11 @@ PostmasterMain(int argc, char *argv[]) */ process_shmem_requests(); + /* + * Give the chance for subsystems to auto-tune their values. + */ + AutotuneShmem(); + /* * Now that loadable modules have had their chance to request additional * shared memory, determine the value of any runtime-computed GUCs that diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index d3c68d8b04c..ec8d61dfb9c 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -109,10 +109,17 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } -Size -AioShmemSize(void) +void +AioAutotune(void) { - Size sz = 0; + char buf[32]; + + if (io_max_concurrency != -1) + return; + + snprintf(buf, sizeof(buf), "%d", AioChooseMaxConcurrency()); + SetConfigOption("io_max_concurrency", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); /* * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. @@ -120,17 +127,15 @@ AioShmemSize(void) * config file, then PGC_S_DYNAMIC_DEFAULT will fail to override that and * we must force the matter with PGC_S_OVERRIDE. */ - if (io_max_concurrency == -1) - { - char buf[32]; - - snprintf(buf, sizeof(buf), "%d", AioChooseMaxConcurrency()); + if (io_max_concurrency == -1) /* failed to apply it? */ SetConfigOption("io_max_concurrency", buf, PGC_POSTMASTER, - PGC_S_DYNAMIC_DEFAULT); - if (io_max_concurrency == -1) /* failed to apply it? */ - SetConfigOption("io_max_concurrency", buf, PGC_POSTMASTER, - PGC_S_OVERRIDE); - } + PGC_S_OVERRIDE); +} + +Size +AioShmemSize(void) +{ + Size sz = 0; sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 85c67b2c183..673a55313f4 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -372,3 +372,25 @@ InitializeShmemGUCs(void) sprintf(buf, "%d", ProcGlobalSemas()); SetConfigOption("num_os_semaphores", buf, PGC_INTERNAL, PGC_S_DYNAMIC_DEFAULT); } + +/* + * Auto-tune shared memory configuration + * + * Some subsystems auto-tune their configurations based on the NBuffers + * value. This must be called before CalculateShmemSize and will change the GUC + * values (when there is one) with the auto-tuned value. + */ +void +AutotuneShmem(void) +{ + Size requested_size; + + AioAutotune(); + CLOGAutotune(); + CommitTsAutotune(); + SUBTRANSAutotune(); + XLOGAutotune(); + + requested_size = CalculateShmemSize(); + BufferManagerAutotune(requested_size); +} diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index e54bf1e760f..0e8744ed37c 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -4133,6 +4133,11 @@ PostgresSingleUserMain(int argc, char *argv[], */ process_shmem_requests(); + /* + * Give the chance for subsystems to auto-tune their values. + */ + AutotuneShmem(); + /* * Now that loadable modules have had their chance to request additional * shared memory, determine the value of any runtime-computed GUCs that diff --git a/src/include/access/clog.h b/src/include/access/clog.h index a1cfed5f43c..b2a9c0ac255 100644 --- a/src/include/access/clog.h +++ b/src/include/access/clog.h @@ -41,6 +41,7 @@ extern void TransactionIdSetTreeStatus(TransactionId xid, int nsubxids, extern XidStatus TransactionIdGetStatus(TransactionId xid, XLogRecPtr *lsn); extern Size CLOGShmemSize(void); +extern void CLOGAutotune(void); extern void CLOGShmemInit(void); extern void BootStrapCLOG(void); extern void StartupCLOG(void); diff --git a/src/include/access/commit_ts.h b/src/include/access/commit_ts.h index bc3b81687b1..80b35d2e0dc 100644 --- a/src/include/access/commit_ts.h +++ b/src/include/access/commit_ts.h @@ -28,6 +28,7 @@ extern TransactionId GetLatestCommitTsData(TimestampTz *ts, RepOriginId *nodeid); extern Size CommitTsShmemSize(void); +extern void CommitTsAutotune(void); extern void CommitTsShmemInit(void); extern void BootStrapCommitTs(void); extern void StartupCommitTs(void); diff --git a/src/include/access/subtrans.h b/src/include/access/subtrans.h index 11b7355dbdf..3772af83f68 100644 --- a/src/include/access/subtrans.h +++ b/src/include/access/subtrans.h @@ -16,6 +16,7 @@ extern TransactionId SubTransGetParent(TransactionId xid); extern TransactionId SubTransGetTopmostTransaction(TransactionId xid); extern Size SUBTRANSShmemSize(void); +extern void SUBTRANSAutotune(void); extern void SUBTRANSShmemInit(void); extern void BootStrapSUBTRANS(void); extern void StartupSUBTRANS(TransactionId oldestActiveXID); diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 0591a885dd1..21741eb9254 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -245,6 +245,7 @@ extern bool DataChecksumsEnabled(void); extern bool GetDefaultCharSignedness(void); extern XLogRecPtr GetFakeLSNForUnloggedRel(void); extern Size XLOGShmemSize(void); +extern void XLOGAutotune(void); extern void XLOGShmemInit(void); extern void BootStrapXLOG(uint32 data_checksum_version); extern void InitializeWalConsistencyChecking(void); diff --git a/src/include/storage/aio_subsys.h b/src/include/storage/aio_subsys.h index 276cb3e31c4..1d305d75000 100644 --- a/src/include/storage/aio_subsys.h +++ b/src/include/storage/aio_subsys.h @@ -21,6 +21,7 @@ /* aio_init.c */ extern Size AioShmemSize(void); +extern void AioAutotune(void); extern void AioShmemInit(void); extern void pgaio_init_backend(void); diff --git a/src/include/storage/ipc.h b/src/include/storage/ipc.h index da32787ab51..f3cbf8abaad 100644 --- a/src/include/storage/ipc.h +++ b/src/include/storage/ipc.h @@ -83,5 +83,6 @@ extern void CreateSharedMemoryAndSemaphores(void); extern void AttachSharedMemoryStructs(void); #endif extern void InitializeShmemGUCs(void); +extern void AutotuneShmem(void); #endif /* IPC_H */ -- 2.52.0