From 9439eb1c5c194d38a6aa388d8d8ca34350ae300f Mon Sep 17 00:00:00 2001 From: diphantxm Date: Thu, 28 Nov 2024 21:24:59 +0300 Subject: [PATCH] parameter max_log_size to truncate logs There is no need to log the entire query, because it may be large and take lots of space on disk. Parameter max_log_size set the maximum length for logged query. Everything beyond that length is truncated. Value 0 disables the parameter. --- doc/src/sgml/config.sgml | 15 ++++++++ src/backend/tcop/postgres.c | 10 ++++- src/backend/utils/error/elog.c | 38 +++++++++++++++++++ src/backend/utils/misc/guc_tables.c | 11 ++++++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/bin/pg_ctl/t/004_logrotate.pl | 15 ++++++++ src/include/utils/elog.h | 2 + 7 files changed, 91 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 76ab72db96..5110b61ba5 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -7913,6 +7913,21 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' + + max_log_size (integer) + + max_log_size configuration parameter + + + + + If greater than zero, each query logged is truncated to this many bytes. + Zero disables the setting. + Default value is 5kB. + + + + diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 4b985bd056..549be13e5b 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -70,6 +70,7 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" +#include "utils/elog.h" #include "utils/guc_hooks.h" #include "utils/injection_point.h" #include "utils/lsyscache.h" @@ -1029,11 +1030,14 @@ exec_simple_query(const char *query_string) bool was_logged = false; bool use_implicit_block; char msec_str[32]; + char* query_log; /* * Report query to various monitoring facilities. */ debug_query_string = query_string; + bool copied = false; + query_log = build_query_log(query_string, &copied); pgstat_report_activity(STATE_RUNNING, query_string); @@ -1078,7 +1082,7 @@ exec_simple_query(const char *query_string) if (check_log_statement(parsetree_list)) { ereport(LOG, - (errmsg("statement: %s", query_string), + (errmsg("statement: %s", query_log), errhidestmt(true), errdetail_execute(parsetree_list))); was_logged = true; @@ -1377,7 +1381,7 @@ exec_simple_query(const char *query_string) case 2: ereport(LOG, (errmsg("duration: %s ms statement: %s", - msec_str, query_string), + msec_str, query_log), errhidestmt(true), errdetail_execute(parsetree_list))); break; @@ -1388,6 +1392,8 @@ exec_simple_query(const char *query_string) TRACE_POSTGRESQL_QUERY_DONE(query_string); + if (query_log && copied) + pfree(query_log); debug_query_string = NULL; } diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 289059435a..aa9712ba64 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -111,6 +111,7 @@ int Log_destination = LOG_DESTINATION_STDERR; char *Log_destination_string = NULL; bool syslog_sequence_numbers = true; bool syslog_split_messages = true; +int max_log_size = 0; /* Processed form of backtrace_functions GUC */ static char *backtrace_function_list; @@ -1693,6 +1694,10 @@ EmitErrorReport(void) CHECK_STACK_DEPTH(); oldcontext = MemoryContextSwitchTo(edata->assoc_context); + const char* old_query_string = debug_query_string; + bool copied = false; + debug_query_string = build_query_log(debug_query_string, &copied); + /* * Reset the formatted timestamp fields before emitting any logs. This * includes all the log destinations and emit_log_hook, as the latter @@ -1733,6 +1738,12 @@ EmitErrorReport(void) MemoryContextSwitchTo(oldcontext); recursion_depth--; + + if (debug_query_string && copied) + { + pfree(debug_query_string); + debug_query_string = old_query_string; + } } /* @@ -3773,3 +3784,30 @@ write_stderr(const char *fmt,...) #endif va_end(ap); } + +/* + * Apply truncation to build query that will be logged. + * + * If query needs to be truncated, copied will be set to true + * and returned string must be freed +*/ +char* +build_query_log(const char* query, bool *copied) +{ + *copied = false; + if (!query) + return NULL; + + size_t query_len = strlen(query); + if (max_log_size == 0 || query_len < max_log_size) + { + return query; + } + + *copied = true; + size_t query_log_len = pg_mbcliplen(query, query_len, max_log_size); + char* query_log = (char*)palloc(query_log_len+1); + memcpy(query_log, query, query_log_len); + query_log[query_log_len] = '\0'; + return query_log; +} diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 9845abd693..64f21377ad 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3724,6 +3724,17 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_log_size", PGC_SUSET, LOGGING_WHAT, + gettext_noop("Sets max size in bytes of logged statement."), + NULL + }, + &max_log_size, + 5 * (1024 * 1024), + 0, INT_MAX, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 407cd1e08c..44971cecc6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -617,6 +617,8 @@ # bind-parameter values to N bytes; # -1 means print in full, 0 disables #log_statement = 'none' # none, ddl, mod, all +#max_log_size = 0 # max size of logged statement_timeout (in bytes) + # 0 disables the feature #log_replication_commands = off #log_temp_files = -1 # log temporary files equal or larger # than the specified size in kilobytes; diff --git a/src/bin/pg_ctl/t/004_logrotate.pl b/src/bin/pg_ctl/t/004_logrotate.pl index eacca1a652..1c1f89cb0a 100644 --- a/src/bin/pg_ctl/t/004_logrotate.pl +++ b/src/bin/pg_ctl/t/004_logrotate.pl @@ -69,6 +69,7 @@ log_destination = 'stderr, csvlog, jsonlog' # these ensure stability of test results: log_rotation_age = 0 lc_messages = 'C' +max_log_size = 32 )); $node->start(); @@ -135,6 +136,20 @@ check_log_pattern('stderr', $new_current_logfiles, 'syntax error', $node); check_log_pattern('csvlog', $new_current_logfiles, 'syntax error', $node); check_log_pattern('jsonlog', $new_current_logfiles, 'syntax error', $node); +$node->psql('postgres', 'INSERT INTO SOME_NON_EXISTANT_TABLE VALUES (TEST)'); +for (my $attempts = 0; $attempts < $max_attempts; $attempts++) +{ + eval { + $current_logfiles = slurp_file($node->data_dir . '/current_logfiles'); + }; + last unless $@; + usleep(100_000); +} +die $@ if $@; +check_log_pattern('stderr', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); +check_log_pattern('csvlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); +check_log_pattern('jsonlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node); + $node->stop(); done_testing(); diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index e54eca5b48..f6689796a5 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -502,6 +502,7 @@ extern PGDLLIMPORT int Log_destination; extern PGDLLIMPORT char *Log_destination_string; extern PGDLLIMPORT bool syslog_sequence_numbers; extern PGDLLIMPORT bool syslog_split_messages; +extern PGDLLIMPORT int max_log_size; /* Log destination bitmap */ #define LOG_DESTINATION_STDERR 1 @@ -516,6 +517,7 @@ extern void log_status_format(StringInfo buf, const char *format, extern void DebugFileOpen(void); extern char *unpack_sql_state(int sql_state); extern bool in_error_recursion_trouble(void); +extern char* build_query_log(const char* query, bool *copied); /* Common functions shared across destinations */ extern void reset_formatted_start_time(void); -- 2.39.2 (Apple Git-143)