On 1/7/26 5:47 PM, Sami Imseih wrote:
Hi,
This seems to have flown under the radar. I still think it's a good
feature even if we have parallel logging in pg_stat_database and
pg_stat_statements (e7a9496de and cf54a2c002)
Thank you for your message.
The latest patchset needs rebasing.
The new patch set is attached.
--
Benoit Lobréau
Consultant
http://dalibo.com
From 0477203b123d96bd5dfa1d47f0509aa1a156ddcd Mon Sep 17 00:00:00 2001
From: benoit <[email protected]>
Date: Tue, 8 Oct 2024 12:39:41 +0200
Subject: [PATCH 1/3] Add a guc for parallel worker logging
The new guc log_parallel_workers controls whether a log message is
produced to display information on the number of workers spawned when a
parallel query or utility is executed.
The default value is `none` which disables logging. `all` displays
information for all parallel queries, whereas `shortage` displays
information only when the number of workers launched is lower than the
number of planned workers.
This new parameter can help database administrators and developers
diagnose performance issues related to parallelism and optimize the
configuration of the system accordingly.
---
doc/src/sgml/config.sgml | 18 ++++++++++++++++++
src/backend/access/transam/parallel.c | 19 +++++++++++++++++++
src/backend/utils/misc/guc_parameters.dat | 8 ++++++++
src/backend/utils/misc/guc_tables.c | 9 +++++++++
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/access/parallel.h | 10 ++++++++++
src/include/utils/guc.h | 1 +
7 files changed, 66 insertions(+)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 0fad34da6eb..5332993631d 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -8066,6 +8066,24 @@ log_line_prefix = '%m [%p] %q%u@%d/%a '
</listitem>
</varlistentry>
+ <varlistentry id="guc-log-parallel-workers" xreflabel="log_parallel_workers">
+ <term><varname>log_parallel_workers</varname> (<type>enum</type>)
+ <indexterm>
+ <primary><varname>log_parallel_workers</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Controls whether a log message about the number of workers is emitted during the
+ execution of a parallel query or utility statement. The default value is
+ <literal>none</literal> which disables logging. <literal>all</literal> emits
+ information for all parallel queries or utilities, whereas <literal>shortage</literal>
+ emits information only when the number of workers launched is lower than the number
+ of planned workers.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-log-parameter-max-length" xreflabel="log_parameter_max_length">
<term><varname>log_parameter_max_length</varname> (<type>integer</type>)
<indexterm>
diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c
index 01a89104ef0..817edfdd62a 100644
--- a/src/backend/access/transam/parallel.c
+++ b/src/backend/access/transam/parallel.c
@@ -1670,3 +1670,22 @@ LookupParallelWorkerFunction(const char *libraryname, const char *funcname)
return (parallel_worker_main_type)
load_external_function(libraryname, funcname, true, NULL);
}
+
+/*
+ * If required, emit information about parallel workers usage in
+ * the logs.
+ */
+void
+LogParallelWorkersIfNeeded(int log_parallel_workers,
+ int parallel_workers_to_launch,
+ int parallel_workers_launched)
+{
+ if ((log_parallel_workers == LOG_PARALLEL_WORKERS_ALL &&
+ parallel_workers_to_launch > 0) ||
+ (log_parallel_workers == LOG_PARALLEL_WORKERS_SHORTAGE &&
+ parallel_workers_to_launch != parallel_workers_launched))
+ ereport(LOG,
+ (errmsg("launched %i parallel workers (planned: %i)",
+ parallel_workers_launched,
+ parallel_workers_to_launch)));
+}
diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat
index 7c60b125564..a096f72fe57 100644
--- a/src/backend/utils/misc/guc_parameters.dat
+++ b/src/backend/utils/misc/guc_parameters.dat
@@ -1700,6 +1700,14 @@
options => 'server_message_level_options',
},
+{ name => 'log_parallel_workers', type => 'enum', context => 'PGC_SUSET', group => 'LOGGING_WHAT',
+ short_desc => 'Log information about parallel worker usage.',
+ long_desc => '"none" doesn\'t log anything, "all" logs all parallel worker usage and "shortage" logs only when the planned workers couldn\'t be acquired',
+ variable => 'log_parallel_workers',
+ boot_val => 'LOG_PARALLEL_WORKERS_NONE',
+ options => 'log_parallel_workers_options',
+},
+
{ name => 'log_parameter_max_length', type => 'int', context => 'PGC_SUSET', group => 'LOGGING_WHAT',
short_desc => 'Sets the maximum length in bytes of data logged for bind parameter values when logging statements.',
long_desc => '-1 means log values in full.',
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 73ff6ad0a32..feee4fee7a6 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -31,6 +31,7 @@
#include "access/commit_ts.h"
#include "access/gin.h"
+#include "access/parallel.h"
#include "access/slru.h"
#include "access/toast_compression.h"
#include "access/twophase.h"
@@ -491,6 +492,13 @@ static const struct config_enum_entry file_copy_method_options[] = {
{NULL, 0, false}
};
+static const struct config_enum_entry log_parallel_workers_options[] = {
+ {"none", LOG_PARALLEL_WORKERS_NONE, false},
+ {"all", LOG_PARALLEL_WORKERS_ALL, false},
+ {"shortage", LOG_PARALLEL_WORKERS_SHORTAGE, false},
+ {NULL, 0, false}
+};
+
/*
* Options for enum values stored in other modules
*/
@@ -544,6 +552,7 @@ int log_min_duration_statement = -1;
int log_parameter_max_length = -1;
int log_parameter_max_length_on_error = 0;
int log_temp_files = -1;
+int log_parallel_workers = LOG_PARALLEL_WORKERS_NONE;
double log_statement_sample_rate = 1.0;
double log_xact_sample_rate = 0;
char *backtrace_functions;
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index dc9e2255f8a..8dcba3aae4a 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -646,6 +646,7 @@
#log_temp_files = -1 # log temporary files equal or larger
# than the specified size in kilobytes;
# -1 disables, 0 logs all temp files
+#log_parallel_workers = none # none, all, shortage
#log_timezone = 'GMT'
# - Process Title -
diff --git a/src/include/access/parallel.h b/src/include/access/parallel.h
index 01bdf2bec1f..623ca839001 100644
--- a/src/include/access/parallel.h
+++ b/src/include/access/parallel.h
@@ -53,6 +53,12 @@ typedef struct ParallelWorkerContext
shm_toc *toc;
} ParallelWorkerContext;
+typedef enum {
+ LOG_PARALLEL_WORKERS_NONE=0,
+ LOG_PARALLEL_WORKERS_ALL,
+ LOG_PARALLEL_WORKERS_SHORTAGE,
+} log_parallel_workers_option_list;
+
extern PGDLLIMPORT volatile sig_atomic_t ParallelMessagePending;
extern PGDLLIMPORT int ParallelWorkerNumber;
extern PGDLLIMPORT bool InitializingParallelWorker;
@@ -78,4 +84,8 @@ extern void ParallelWorkerReportLastRecEnd(XLogRecPtr last_xlog_end);
extern void ParallelWorkerMain(Datum main_arg);
+extern void LogParallelWorkersIfNeeded(int log_parallel_workers,
+ int parallel_workers_to_launch,
+ int parallel_workers_launched);
+
#endif /* PARALLEL_H */
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index bf39878c43e..aee4012950d 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -303,6 +303,7 @@ extern PGDLLIMPORT int log_temp_files;
extern PGDLLIMPORT double log_statement_sample_rate;
extern PGDLLIMPORT double log_xact_sample_rate;
extern PGDLLIMPORT char *backtrace_functions;
+extern PGDLLIMPORT int log_parallel_workers;
extern PGDLLIMPORT int temp_file_limit;
--
2.47.3
From 5dfcc90e1afbe70a1f5158938d5b765f50eda209 Mon Sep 17 00:00:00 2001
From: benoit <[email protected]>
Date: Wed, 29 Jan 2025 17:10:57 +0100
Subject: [PATCH 2/3] Implements logging for parallel worker usage in utilities
This patch implements logging of parallel worker usage for:
* the index cleanup and bulkdelete phases of vacuum;
* btree, brin and gin index builds.
---
src/backend/access/brin/brin.c | 4 ++++
src/backend/access/gin/gininsert.c | 4 ++++
src/backend/access/nbtree/nbtsort.c | 4 ++++
src/backend/commands/vacuumparallel.c | 13 +++++++++++++
4 files changed, 25 insertions(+)
diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c
index 6887e421442..de2d7dad5c8 100644
--- a/src/backend/access/brin/brin.c
+++ b/src/backend/access/brin/brin.c
@@ -2567,6 +2567,10 @@ _brin_end_parallel(BrinLeader *brinleader, BrinBuildState *state)
/* Shutdown worker processes */
WaitForParallelWorkersToFinish(brinleader->pcxt);
+ LogParallelWorkersIfNeeded(log_parallel_workers,
+ brinleader->pcxt->nworkers_to_launch,
+ brinleader->pcxt->nworkers_launched);
+
/*
* Next, accumulate WAL usage. (This must wait for the workers to finish,
* or we might get incomplete data.)
diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c
index c08ea927ac5..dd5da905927 100644
--- a/src/backend/access/gin/gininsert.c
+++ b/src/backend/access/gin/gininsert.c
@@ -1110,6 +1110,10 @@ _gin_end_parallel(GinLeader *ginleader, GinBuildState *state)
/* Shutdown worker processes */
WaitForParallelWorkersToFinish(ginleader->pcxt);
+ LogParallelWorkersIfNeeded(log_parallel_workers,
+ ginleader->pcxt->nworkers_to_launch,
+ ginleader->pcxt->nworkers_launched);
+
/*
* Next, accumulate WAL usage. (This must wait for the workers to finish,
* or we might get incomplete data.)
diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c
index 90ab4e91b56..9500fb72b91 100644
--- a/src/backend/access/nbtree/nbtsort.c
+++ b/src/backend/access/nbtree/nbtsort.c
@@ -1612,6 +1612,10 @@ _bt_end_parallel(BTLeader *btleader)
/* Shutdown worker processes */
WaitForParallelWorkersToFinish(btleader->pcxt);
+ LogParallelWorkersIfNeeded(log_parallel_workers,
+ btleader->pcxt->nworkers_to_launch,
+ btleader->pcxt->nworkers_launched);
+
/*
* Next, accumulate WAL usage. (This must wait for the workers to finish,
* or we might get incomplete data.)
diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c
index c3b3c9ea21a..ab1db352511 100644
--- a/src/backend/commands/vacuumparallel.c
+++ b/src/backend/commands/vacuumparallel.c
@@ -208,6 +208,9 @@ struct ParallelVacuumState
int nindexes_parallel_cleanup;
int nindexes_parallel_condcleanup;
+ int nworkers_to_launch;
+ int nworkers_launched;
+
/* Buffer access strategy used by leader process */
BufferAccessStrategy bstrategy;
@@ -362,6 +365,9 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes,
if ((vacoptions & VACUUM_OPTION_PARALLEL_COND_CLEANUP) != 0)
pvs->nindexes_parallel_condcleanup++;
}
+ pvs->nworkers_to_launch = 0;
+ pvs->nworkers_launched = 0;
+
shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_INDEX_STATS, indstats);
pvs->indstats = indstats;
@@ -437,6 +443,10 @@ parallel_vacuum_end(ParallelVacuumState *pvs, IndexBulkDeleteResult **istats)
{
Assert(!IsParallelWorker());
+ LogParallelWorkersIfNeeded(log_parallel_workers,
+ pvs->nworkers_to_launch,
+ pvs->nworkers_launched);
+
/* Copy the updated statistics */
for (int i = 0; i < pvs->nindexes; i++)
{
@@ -738,6 +748,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan
for (int i = 0; i < pvs->pcxt->nworkers_launched; i++)
InstrAccumParallelQuery(&pvs->buffer_usage[i], &pvs->wal_usage[i]);
+
+ pvs->nworkers_to_launch += pvs->pcxt->nworkers_to_launch;
+ pvs->nworkers_launched += pvs->pcxt->nworkers_launched;
}
/*
--
2.47.3
From 3c1923e012f59f14074157011300234d0f01394c Mon Sep 17 00:00:00 2001
From: benoit <[email protected]>
Date: Wed, 29 Jan 2025 17:15:25 +0100
Subject: [PATCH 3/3] Implements logging for parallel worker usage in queries
---
src/backend/executor/execMain.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index ca14cdabdd0..fb4eeffdaa3 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -488,6 +488,10 @@ standard_ExecutorEnd(QueryDesc *queryDesc)
pgstat_update_parallel_workers_stats((PgStat_Counter) estate->es_parallel_workers_to_launch,
(PgStat_Counter) estate->es_parallel_workers_launched);
+ LogParallelWorkersIfNeeded(log_parallel_workers,
+ estate->es_parallel_workers_to_launch,
+ estate->es_parallel_workers_launched);
+
/*
* Check that ExecutorFinish was called, unless in EXPLAIN-only mode. This
* Assert is needed because ExecutorFinish is new as of 9.1, and callers
--
2.47.3
