On Thu, Dec 09, 2021 at 09:53:23AM -0600, Justin Pryzby wrote:
> On Thu, Dec 09, 2021 at 05:17:54PM +0900, Michael Paquier wrote:
> > On Wed, Dec 08, 2021 at 01:23:51PM +0100, Peter Eisentraut wrote:
> > > I wasn't really aware of this script either. But I think it's a good idea
> > > to have it. But only if it's run automatically as part of a test suite
> > > run.
> >
> > Okay. If we do that, I am wondering whether it would be better to
> > rewrite this script in perl then, so as there is no need to worry
> > about the compatibility of grep. And also, it would make sense to
> > return a non-zero exit code if an incompatibility is found for the
> > automation part.
>
> One option is to expose the GUC flags in pg_settings, so this can all be done
> in SQL regression tests.
>
> Maybe the flags should be text strings, so it's a nicer user-facing interface.
> But then the field would be pretty wide, even though we're only adding it for
> regression tests. The only other alternative I can think of is to make a
> sql-callable function like pg_get_guc_flags(text guc).
Fixed regression tests caused by another patches.
>From f18854a561d779c3d77689f9778dc175db1a2349 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Sun, 5 Dec 2021 23:22:04 -0600
Subject: [PATCH v2 1/2] check_guc: fix absurd number of false positives
Avoid false positives;
Avoid various assumptions;
Avoid list of exceptions;
Simplify shell/awk/sed/grep;
This requires GNU awk for RS as a regex.
---
src/backend/utils/misc/check_guc | 69 +++++---------------------------
1 file changed, 10 insertions(+), 59 deletions(-)
diff --git a/src/backend/utils/misc/check_guc b/src/backend/utils/misc/check_guc
index b171ef0e4f..323ca13191 100755
--- a/src/backend/utils/misc/check_guc
+++ b/src/backend/utils/misc/check_guc
@@ -1,78 +1,29 @@
-#!/bin/sh
+#! /bin/sh
+set -e
-## currently, this script makes a lot of assumptions:
+## this script makes some assumptions about the formatting of files it parses.
## in postgresql.conf.sample:
## 1) the valid config settings may be preceded by a '#', but NOT '# '
## (we use this to skip comments)
-## 2) the valid config settings will be followed immediately by ' ='
-## (at least one space preceding the '=')
-## in guc.c:
-## 3) the options have PGC_ on the same line as the option
-## 4) the options have '{' on the same line as the option
-
-## Problems
-## 1) Don't know what to do with TRANSACTION ISOLATION LEVEL
-
-## if an option is valid but shows up in only one file (guc.c but not
-## postgresql.conf.sample), it should be listed here so that it
-## can be ignored
-INTENTIONALLY_NOT_INCLUDED="debug_deadlocks in_hot_standby \
-is_superuser lc_collate lc_ctype lc_messages lc_monetary lc_numeric lc_time \
-pre_auth_delay role seed server_encoding server_version server_version_num \
-session_authorization trace_lock_oidmin trace_lock_table trace_locks trace_lwlocks \
-trace_notify trace_userlocks transaction_isolation transaction_read_only \
-zero_damaged_pages"
### What options are listed in postgresql.conf.sample, but don't appear
### in guc.c?
-# grab everything that looks like a setting and convert it to lower case
-SETTINGS=`grep ' =' postgresql.conf.sample |
-grep -v '^# ' | # strip comments
-sed -e 's/^#//' |
-awk '{print $1}'`
-
-SETTINGS=`echo "$SETTINGS" | tr 'A-Z' 'a-z'`
+# grab everything that looks like a setting
+SETTINGS=`sed '/^#[[:alnum:]]/!d; s/^#//; s/ =.*//; /^include/d' postgresql.conf.sample`
for i in $SETTINGS ; do
- hidden=0
## it sure would be nice to replace this with an sql "not in" statement
- ## it doesn't seem to make sense to have things in .sample and not in guc.c
-# for hidethis in $INTENTIONALLY_NOT_INCLUDED ; do
-# if [ "$hidethis" = "$i" ] ; then
-# hidden=1
-# fi
-# done
- if [ "$hidden" -eq 0 ] ; then
- grep -i '"'$i'"' guc.c > /dev/null
- if [ $? -ne 0 ] ; then
- echo "$i seems to be missing from guc.c";
- fi;
- fi
+ grep -i "\"$i\"" guc.c >/dev/null ||
+ echo "$i seems to be missing from guc.c";
done
### What options are listed in guc.c, but don't appear
### in postgresql.conf.sample?
# grab everything that looks like a setting and convert it to lower case
-
-SETTINGS=`grep '{.* PGC_' guc.c | awk '{print $1}' | \
- sed -e 's/{//g' -e 's/"//g' -e 's/,//'`
-
-SETTINGS=`echo "$SETTINGS" | tr 'A-Z' 'a-z'`
-
+SETTINGS=`gawk -F '[",]' 'BEGIN{RS="\n\t\\\\{\n"} /",[[:space:]]*PGC_.*.*gettext_noop/ && !/NOT_IN_SAMPLE/{print tolower($2)}' guc.c`
for i in $SETTINGS ; do
- hidden=0
- ## it sure would be nice to replace this with an sql "not in" statement
- for hidethis in $INTENTIONALLY_NOT_INCLUDED ; do
- if [ "$hidethis" = "$i" ] ; then
- hidden=1
- fi
- done
- if [ "$hidden" -eq 0 ] ; then
- grep -i '#'$i' ' postgresql.conf.sample > /dev/null
- if [ $? -ne 0 ] ; then
- echo "$i seems to be missing from postgresql.conf.sample";
- fi
- fi
+ grep "#$i " postgresql.conf.sample >/dev/null ||
+ echo "$i seems to be missing from postgresql.conf.sample";
done
--
2.17.0
>From ffaffb0350678c7c774aad0b05a7b49f301c08e6 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Wed, 8 Dec 2021 12:06:35 -0600
Subject: [PATCH v2 2/2] Expose GUC flags in pg_settings; retire ./check_guc
---
src/backend/utils/misc/check_guc | 29 -------
src/backend/utils/misc/guc.c | 8 +-
src/include/catalog/pg_proc.dat | 6 +-
src/test/regress/expected/guc.out | 114 ++++++++++++++++++++++++++++
src/test/regress/expected/rules.out | 5 +-
src/test/regress/sql/guc.sql | 63 +++++++++++++++
6 files changed, 190 insertions(+), 35 deletions(-)
delete mode 100755 src/backend/utils/misc/check_guc
diff --git a/src/backend/utils/misc/check_guc b/src/backend/utils/misc/check_guc
deleted file mode 100755
index 323ca13191..0000000000
--- a/src/backend/utils/misc/check_guc
+++ /dev/null
@@ -1,29 +0,0 @@
-#! /bin/sh
-set -e
-
-## this script makes some assumptions about the formatting of files it parses.
-## in postgresql.conf.sample:
-## 1) the valid config settings may be preceded by a '#', but NOT '# '
-## (we use this to skip comments)
-
-### What options are listed in postgresql.conf.sample, but don't appear
-### in guc.c?
-
-# grab everything that looks like a setting
-SETTINGS=`sed '/^#[[:alnum:]]/!d; s/^#//; s/ =.*//; /^include/d' postgresql.conf.sample`
-
-for i in $SETTINGS ; do
- ## it sure would be nice to replace this with an sql "not in" statement
- grep -i "\"$i\"" guc.c >/dev/null ||
- echo "$i seems to be missing from guc.c";
-done
-
-### What options are listed in guc.c, but don't appear
-### in postgresql.conf.sample?
-
-# grab everything that looks like a setting and convert it to lower case
-SETTINGS=`gawk -F '[",]' 'BEGIN{RS="\n\t\\\\{\n"} /",[[:space:]]*PGC_.*.*gettext_noop/ && !/NOT_IN_SAMPLE/{print tolower($2)}' guc.c`
-for i in $SETTINGS ; do
- grep "#$i " postgresql.conf.sample >/dev/null ||
- echo "$i seems to be missing from postgresql.conf.sample";
-done
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index f736e8d872..7d4e2e0374 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -9889,6 +9889,10 @@ GetConfigOptionByNum(int varnum, const char **values, bool *noshow)
}
values[16] = (conf->status & GUC_PENDING_RESTART) ? "t" : "f";
+
+ /* flags */
+ snprintf(buffer, sizeof(buffer), "%d", conf->flags);
+ values[17] = pstrdup(buffer);
}
/*
@@ -9944,7 +9948,7 @@ show_config_by_name_missing_ok(PG_FUNCTION_ARGS)
* show_all_settings - equiv to SHOW ALL command but implemented as
* a Table Function.
*/
-#define NUM_PG_SETTINGS_ATTS 17
+#define NUM_PG_SETTINGS_ATTS 18
Datum
show_all_settings(PG_FUNCTION_ARGS)
@@ -10006,6 +10010,8 @@ show_all_settings(PG_FUNCTION_ARGS)
INT4OID, -1, 0);
TupleDescInitEntry(tupdesc, (AttrNumber) 17, "pending_restart",
BOOLOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 18, "flags",
+ INT4OID, -1, 0);
/*
* Generate attribute metadata needed later to produce tuples from raw
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 4d992dc224..ee86d86905 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -6092,9 +6092,9 @@
{ oid => '2084', descr => 'SHOW ALL as a function',
proname => 'pg_show_all_settings', prorows => '1000', proretset => 't',
provolatile => 's', prorettype => 'record', proargtypes => '',
- proallargtypes => '{text,text,text,text,text,text,text,text,text,text,text,_text,text,text,text,int4,bool}',
- proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
- proargnames => '{name,setting,unit,category,short_desc,extra_desc,context,vartype,source,min_val,max_val,enumvals,boot_val,reset_val,sourcefile,sourceline,pending_restart}',
+ proallargtypes => '{text,text,text,text,text,text,text,text,text,text,text,_text,text,text,text,int4,bool,int4}',
+ proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
+ proargnames => '{name,setting,unit,category,short_desc,extra_desc,context,vartype,source,min_val,max_val,enumvals,boot_val,reset_val,sourcefile,sourceline,pending_restart,flags}',
prosrc => 'show_all_settings' },
{ oid => '3329', descr => 'show config file settings',
proname => 'pg_show_all_file_settings', prorows => '1000', proretset => 't',
diff --git a/src/test/regress/expected/guc.out b/src/test/regress/expected/guc.out
index 2433038c3e..681cf742d1 100644
--- a/src/test/regress/expected/guc.out
+++ b/src/test/regress/expected/guc.out
@@ -832,3 +832,117 @@ ERROR: unrecognized configuration parameter "foo"
RESET plpgsql.extra_foo_warnings;
WARNING: unrecognized configuration parameter "plpgsql.extra_foo_warnings"
DETAIL: "plpgsql" is a reserved prefix.
+--
+-- Test GUC categories and flags.
+--
+-- test that GUCS are in postgresql.conf
+SELECT lower(name) FROM pg_settings WHERE (flags&32)=0 EXCEPT
+SELECT regexp_replace(ln, '^#?([_[:alpha:]]+) = .*', '\1') AS guc
+FROM (SELECT regexp_split_to_table(pg_read_file('postgresql.conf'), '\n') AS ln) conf
+WHERE ln~'^#?[[:alpha:]]'
+ORDER BY 1;
+ lower
+-----------------------------
+ config_file
+ plpgsql.check_asserts
+ plpgsql.extra_errors
+ plpgsql.extra_warnings
+ plpgsql.print_strict_params
+ plpgsql.variable_conflict
+(6 rows)
+
+-- test that lines in postgresql.conf that look like GUCs are GUCs
+SELECT regexp_replace(ln, '^#?([_[:alpha:]]+) = .*', '\1') AS guc
+FROM (SELECT regexp_split_to_table(pg_read_file('postgresql.conf'), '\n') AS ln) conf
+WHERE ln~'^#?[[:alpha:]]'
+EXCEPT SELECT lower(name) FROM pg_settings WHERE (flags&32)=0
+ORDER BY 1;
+ guc
+-------------------
+ include
+ include_dir
+ include_if_exists
+(3 rows)
+
+-- test that section:DEVELOPER GUCs are flagged GUC_NOT_IN_SAMPLE:
+SELECT * FROM (SELECT name, category, (flags&32) != 0 AS not_in_sample FROM pg_settings)x
+WHERE category='Developer Options' AND NOT not_in_sample
+ORDER BY 1;
+ name | category | not_in_sample
+------+----------+---------------
+(0 rows)
+
+-- Maybe the converse:
+SELECT * FROM (SELECT name, category, (flags&32) != 0 AS not_in_sample FROM pg_settings)x
+WHERE category NOT IN ('Developer Options', 'Preset Options') AND not_in_sample
+ORDER BY 1;
+ name | category | not_in_sample
+------------------------+-------------------------------------------------+---------------
+ application_name | Reporting and Logging / What to Log | t
+ transaction_deferrable | Client Connection Defaults / Statement Behavior | t
+ transaction_isolation | Client Connection Defaults / Statement Behavior | t
+ transaction_read_only | Client Connection Defaults / Statement Behavior | t
+(4 rows)
+
+-- Most Query Tuning GUCs are flagged as EXPLAIN:
+SELECT * FROM (SELECT name, category, (flags&1048576) != 0 AS guc_explain FROM pg_settings)x
+WHERE category~'^Query Tuning' AND NOT guc_explain
+ORDER BY 1;
+ name | category | guc_explain
+---------------------------+--------------------------------------+-------------
+ default_statistics_target | Query Tuning / Other Planner Options | f
+(1 row)
+
+-- Maybe the converse:
+SELECT * FROM (SELECT name, category, (flags&1048576) != 0 AS guc_explain FROM pg_settings)x
+WHERE guc_explain AND NOT category~'^Query Tuning|^Resource Usage'
+ORDER BY 1;
+ name | category | guc_explain
+---------------------+-------------------------------------------------+-------------
+ force_parallel_mode | Developer Options | t
+ search_path | Client Connection Defaults / Statement Behavior | t
+(2 rows)
+
+-- GUCs flagged RUNTIME are Preset
+SELECT * FROM (SELECT name, category, (flags&2097152) != 0 AS guc_computed FROM pg_settings)x
+WHERE guc_computed AND NOT category='Preset Options'
+ORDER BY 1;
+ name | category | guc_computed
+------+----------+--------------
+(0 rows)
+
+-- PRESET GUCs are flagged NOT_IN_SAMPLE
+SELECT * FROM (SELECT name, category, (flags&32) != 0 AS not_in_sample FROM pg_settings)x
+WHERE category='Preset Options' AND NOT not_in_sample
+ORDER BY 1;
+ name | category | not_in_sample
+------+----------+---------------
+(0 rows)
+
+-- NO_SHOW_ALL implies NO_RESET_ALL:
+SELECT * FROM (SELECT name, category, (flags&4)!=0 AS no_show_all, (flags&8)!=0 AS no_reset_all FROM pg_settings)x
+WHERE no_show_all AND NOT no_reset_all
+ORDER BY 1;
+ name | category | no_show_all | no_reset_all
+------+----------+-------------+--------------
+(0 rows)
+
+-- Usually the converse:
+SELECT * FROM (SELECT name, category, (flags&4)!=0 AS no_show_all, (flags&8)!=0 AS no_reset_all FROM pg_settings)x
+WHERE NOT no_show_all AND no_reset_all
+ORDER BY 1;
+ name | category | no_show_all | no_reset_all
+------------------------+-------------------------------------------------+-------------+--------------
+ transaction_deferrable | Client Connection Defaults / Statement Behavior | f | t
+ transaction_isolation | Client Connection Defaults / Statement Behavior | f | t
+ transaction_read_only | Client Connection Defaults / Statement Behavior | f | t
+(3 rows)
+
+-- NO_SHOW_ALL implies NOT_IN_SAMPLE:
+SELECT * FROM (SELECT name, category, (flags&4)!=0 AS no_show_all, (flags&32) != 0 AS not_in_sample FROM pg_settings)x
+WHERE no_show_all AND NOT not_in_sample
+ORDER BY 1;
+ name | category | no_show_all | not_in_sample
+------+----------+-------------+---------------
+(0 rows)
+
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b58b062b10..2db00f2a8d 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1724,8 +1724,9 @@ pg_settings| SELECT a.name,
a.reset_val,
a.sourcefile,
a.sourceline,
- a.pending_restart
- FROM pg_show_all_settings() a(name, setting, unit, category, short_desc, extra_desc, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, sourcefile, sourceline, pending_restart);
+ a.pending_restart,
+ a.flags
+ FROM pg_show_all_settings() a(name, setting, unit, category, short_desc, extra_desc, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, sourcefile, sourceline, pending_restart, flags);
pg_shadow| SELECT pg_authid.rolname AS usename,
pg_authid.oid AS usesysid,
pg_authid.rolcreatedb AS usecreatedb,
diff --git a/src/test/regress/sql/guc.sql b/src/test/regress/sql/guc.sql
index b57758ed27..0da67b9582 100644
--- a/src/test/regress/sql/guc.sql
+++ b/src/test/regress/sql/guc.sql
@@ -322,3 +322,66 @@ SHOW plpgsql.extra_foo_warnings; -- but the parameter is set
-- cleanup
RESET foo;
RESET plpgsql.extra_foo_warnings;
+
+--
+-- Test GUC categories and flags.
+--
+
+-- test that GUCS are in postgresql.conf
+SELECT lower(name) FROM pg_settings WHERE (flags&32)=0 EXCEPT
+SELECT regexp_replace(ln, '^#?([_[:alpha:]]+) = .*', '\1') AS guc
+FROM (SELECT regexp_split_to_table(pg_read_file('postgresql.conf'), '\n') AS ln) conf
+WHERE ln~'^#?[[:alpha:]]'
+ORDER BY 1;
+
+-- test that lines in postgresql.conf that look like GUCs are GUCs
+SELECT regexp_replace(ln, '^#?([_[:alpha:]]+) = .*', '\1') AS guc
+FROM (SELECT regexp_split_to_table(pg_read_file('postgresql.conf'), '\n') AS ln) conf
+WHERE ln~'^#?[[:alpha:]]'
+EXCEPT SELECT lower(name) FROM pg_settings WHERE (flags&32)=0
+ORDER BY 1;
+
+-- test that section:DEVELOPER GUCs are flagged GUC_NOT_IN_SAMPLE:
+SELECT * FROM (SELECT name, category, (flags&32) != 0 AS not_in_sample FROM pg_settings)x
+WHERE category='Developer Options' AND NOT not_in_sample
+ORDER BY 1;
+
+-- Maybe the converse:
+SELECT * FROM (SELECT name, category, (flags&32) != 0 AS not_in_sample FROM pg_settings)x
+WHERE category NOT IN ('Developer Options', 'Preset Options') AND not_in_sample
+ORDER BY 1;
+
+-- Most Query Tuning GUCs are flagged as EXPLAIN:
+SELECT * FROM (SELECT name, category, (flags&1048576) != 0 AS guc_explain FROM pg_settings)x
+WHERE category~'^Query Tuning' AND NOT guc_explain
+ORDER BY 1;
+
+-- Maybe the converse:
+SELECT * FROM (SELECT name, category, (flags&1048576) != 0 AS guc_explain FROM pg_settings)x
+WHERE guc_explain AND NOT category~'^Query Tuning|^Resource Usage'
+ORDER BY 1;
+
+-- GUCs flagged RUNTIME are Preset
+SELECT * FROM (SELECT name, category, (flags&2097152) != 0 AS guc_computed FROM pg_settings)x
+WHERE guc_computed AND NOT category='Preset Options'
+ORDER BY 1;
+
+-- PRESET GUCs are flagged NOT_IN_SAMPLE
+SELECT * FROM (SELECT name, category, (flags&32) != 0 AS not_in_sample FROM pg_settings)x
+WHERE category='Preset Options' AND NOT not_in_sample
+ORDER BY 1;
+
+-- NO_SHOW_ALL implies NO_RESET_ALL:
+SELECT * FROM (SELECT name, category, (flags&4)!=0 AS no_show_all, (flags&8)!=0 AS no_reset_all FROM pg_settings)x
+WHERE no_show_all AND NOT no_reset_all
+ORDER BY 1;
+
+-- Usually the converse:
+SELECT * FROM (SELECT name, category, (flags&4)!=0 AS no_show_all, (flags&8)!=0 AS no_reset_all FROM pg_settings)x
+WHERE NOT no_show_all AND no_reset_all
+ORDER BY 1;
+
+-- NO_SHOW_ALL implies NOT_IN_SAMPLE:
+SELECT * FROM (SELECT name, category, (flags&4)!=0 AS no_show_all, (flags&32) != 0 AS not_in_sample FROM pg_settings)x
+WHERE no_show_all AND NOT not_in_sample
+ORDER BY 1;
--
2.17.0
