From 82ba847cdbd1074b8f630827b20eedefc6a451b9 Mon Sep 17 00:00:00 2001 From: Tatsuro Yamada Date: Mon, 10 Feb 2025 08:33:23 +0900 Subject: [PATCH 1/3] Add a new option STATS to EXPLAIN command This patch allows to show applied extended statistics in EXPLAIN command output. It includes the following points: - Rebased on 9926f854 - Added a new struct Applied_ExtStats in plannode.h (T4) - Hopefully this will solve the issue related to "-DCOPY_PARSE_PLAN_TREES" with cfbot. - To pass the extended statistics information from path to plan, it might be more appropriate to define a new structure in primnode.h rather than plannode.h. Any advice would be appreciated. - Handled EXPLAIN(STATS, VERBOSE) option (M2) - Before the fix, schema name was always added to extended statistics name, but with this patch, schema name is added to the following only when VERBOSE option is selected: - Extended statistics name, table name, and column name - Added Supported extended statistics types in document (M4) --- doc/src/sgml/ref/explain.sgml | 14 +++ src/backend/commands/explain.c | 140 ++++++++++++++++++++++ src/backend/nodes/makefuncs.c | 11 ++ src/backend/optimizer/plan/createplan.c | 17 +++ src/backend/optimizer/util/relnode.c | 12 ++ src/backend/optimizer/util/restrictinfo.c | 35 ++++++ src/backend/statistics/extended_stats.c | 8 ++ src/backend/utils/adt/selfuncs.c | 15 +++ src/backend/utils/cache/lsyscache.c | 49 ++++++++ src/include/commands/explain.h | 1 + src/include/nodes/makefuncs.h | 2 + src/include/nodes/parsenodes.h | 3 + src/include/nodes/pathnodes.h | 5 + src/include/nodes/plannodes.h | 15 +++ src/include/optimizer/restrictinfo.h | 2 + src/include/utils/lsyscache.h | 3 + 16 files changed, 332 insertions(+) diff --git a/doc/src/sgml/ref/explain.sgml b/doc/src/sgml/ref/explain.sgml index 6361a14e65d..176ff4b0f2f 100644 --- a/doc/src/sgml/ref/explain.sgml +++ b/doc/src/sgml/ref/explain.sgml @@ -43,6 +43,7 @@ EXPLAIN [ ( option [, ...] ) ] boolean ] SERIALIZE [ { NONE | TEXT | BINARY } ] WAL [ boolean ] + STATS [ boolean ] TIMING [ boolean ] SUMMARY [ boolean ] MEMORY [ boolean ] @@ -250,6 +251,19 @@ ROLLBACK; + + STATS + + + Include information on applied Extended statistics. + Specifically, include the names of extended statistics and clauses. + Supported extended statistics types are Dependencies and MCV. + See for details about extended + statistics. This parameter defaults to FALSE. + + + + TIMING diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index c24e66f82e1..8aac5736864 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -94,6 +94,15 @@ static void show_qual(List *qual, const char *qlabel, static void show_scan_qual(List *qual, const char *qlabel, PlanState *planstate, List *ancestors, ExplainState *es); +static char *deparse_stat_expression(Node *node, + PlanState *planstate, List *ancestors, + bool useprefix, ExplainState *es); +static char *show_stat_qual(List *qual, int is_or, + PlanState *planstate, List *ancestors, + bool useprefix, ExplainState *es); +static void show_scan_stats(List *stats, List *clauses, List *ors, + PlanState *planstate, List *ancestors, + ExplainState *es); static void show_upper_qual(List *qual, const char *qlabel, PlanState *planstate, List *ancestors, ExplainState *es); @@ -223,6 +232,8 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, es->settings = defGetBoolean(opt); else if (strcmp(opt->defname, "generic_plan") == 0) es->generic = defGetBoolean(opt); + else if (strcmp(opt->defname, "stats") == 0) + es->stats = defGetBoolean(opt); else if (strcmp(opt->defname, "timing") == 0) { timing_set = true; @@ -497,6 +508,11 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, if (es->buffers) bufusage_start = pgBufferUsage; + + /* if this flag is true, applied ext stats are stored */ + if (es->stats) + query->isExplain_Stats = true; + INSTR_TIME_SET_CURRENT(planstart); /* plan the query */ @@ -2114,6 +2130,11 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + if (es->stats) + show_scan_stats(plan->app_extstats->applied_stats, + plan->app_extstats->applied_clauses, + plan->app_extstats->applied_clauses_or, + planstate, ancestors, es); break; case T_IndexOnlyScan: show_scan_qual(((IndexOnlyScan *) plan)->indexqual, @@ -2130,10 +2151,20 @@ ExplainNode(PlanState *planstate, List *ancestors, if (es->analyze) ExplainPropertyFloat("Heap Fetches", NULL, planstate->instrument->ntuples2, 0, es); + if (es->stats) + show_scan_stats(plan->app_extstats->applied_stats, + plan->app_extstats->applied_clauses, + plan->app_extstats->applied_clauses_or, + planstate, ancestors, es); break; case T_BitmapIndexScan: show_scan_qual(((BitmapIndexScan *) plan)->indexqualorig, "Index Cond", planstate, ancestors, es); + if (es->stats) + show_scan_stats(plan->app_extstats->applied_stats, + plan->app_extstats->applied_clauses, + plan->app_extstats->applied_clauses_or, + planstate, ancestors, es); break; case T_BitmapHeapScan: show_scan_qual(((BitmapHeapScan *) plan)->bitmapqualorig, @@ -2164,6 +2195,12 @@ ExplainNode(PlanState *planstate, List *ancestors, planstate, es); if (IsA(plan, CteScan)) show_ctescan_info(castNode(CteScanState, planstate), es); + + if (es->stats) + show_scan_stats(plan->app_extstats->applied_stats, + plan->app_extstats->applied_clauses, + plan->app_extstats->applied_clauses_or, + planstate, ancestors, es); break; case T_Gather: { @@ -2231,6 +2268,11 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + if (es->stats) + show_scan_stats(plan->app_extstats->applied_stats, + plan->app_extstats->applied_clauses, + plan->app_extstats->applied_clauses_or, + planstate, ancestors, es); break; case T_TableFuncScan: if (es->verbose) @@ -2345,6 +2387,11 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + if (es->stats) + show_scan_stats(plan->app_extstats->applied_stats, + plan->app_extstats->applied_clauses, + plan->app_extstats->applied_clauses_or, + planstate, ancestors, es); break; case T_WindowAgg: show_upper_qual(plan->qual, "Filter", planstate, ancestors, es); @@ -2361,6 +2408,11 @@ ExplainNode(PlanState *planstate, List *ancestors, if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); + if (es->stats) + show_scan_stats(plan->app_extstats->applied_stats, + plan->app_extstats->applied_clauses, + plan->app_extstats->applied_clauses_or, + planstate, ancestors, es); break; case T_Sort: show_sort_keys(castNode(SortState, planstate), ancestors, es); @@ -2687,6 +2739,94 @@ show_scan_qual(List *qual, const char *qlabel, show_qual(qual, qlabel, planstate, ancestors, useprefix, es); } +/* + * Show a generic expression + */ +static char * +deparse_stat_expression(Node *node, + PlanState *planstate, List *ancestors, + bool useprefix, ExplainState *es) +{ + List *context; + + /* Set up deparsing context */ + context = set_deparse_context_plan(es->deparse_cxt, + planstate->plan, + ancestors); + + /* Deparse the expression */ + return deparse_expression(node, context, useprefix, false); +} + +/* + * Show a qualifier expression for extended stats + */ +static char * +show_stat_qual(List *qual, int is_or, + PlanState *planstate, List *ancestors, + bool useprefix, ExplainState *es) +{ + Node *node; + + /* No work if empty qual */ + if (qual == NIL) + return NULL; + + /* Convert AND list to explicit AND */ + switch (is_or) + { + case 0: + node = (Node *) make_ands_explicit(qual); + break; + case 1: + node = (Node *) make_ors_explicit(qual); + break; + case 2: + /* Extended stats for GROUP BY clause should be comma separeted string */ + node = (Node *) qual; + break; + default: + elog(ERROR, "unexpected value: %d", is_or); + break; + } + + /* And show it */ + return deparse_stat_expression(node, planstate, ancestors, useprefix, es); +} + +/* + * Show applied statistics for scan/agg/group plan node + */ +static void +show_scan_stats(List *stats, List *clauses, List *ors, + PlanState *planstate, List *ancestors, ExplainState *es) +{ + ListCell *lc1, *lc2, *lc3; + StringInfoData str; + bool useprefix; + + useprefix = es->verbose; + + forthree (lc1, stats, lc2, clauses, lc3, ors) + { + StatisticExtInfo *stat = (StatisticExtInfo *) lfirst(lc1); + List *applied_clauses = (List *) lfirst(lc2); + int is_or = lfirst_int(lc3); + + initStringInfo(&str); + + if (useprefix) + appendStringInfo(&str, "%s.", + get_namespace_name(get_statistics_namespace(stat->statOid))); + + appendStringInfo(&str, "%s Clauses: %s", + get_statistics_name(stat->statOid), + show_stat_qual(applied_clauses, is_or, planstate, ancestors, useprefix, es)); + + ExplainPropertyText("Ext Stats", str.data, es); + } +} + /* * Show a qualifier expression for an upper-level plan node */ diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c index 007612563ca..c09667a8b42 100644 --- a/src/backend/nodes/makefuncs.c +++ b/src/backend/nodes/makefuncs.c @@ -759,6 +759,17 @@ make_ands_explicit(List *andclauses) return make_andclause(andclauses); } +Expr * +make_ors_explicit(List *orclauses) +{ + if (orclauses == NIL) + return (Expr *) makeBoolConst(true, false); + else if (list_length(orclauses) == 1) + return (Expr *) linitial(orclauses); + else + return make_orclause(orclauses); +} + List * make_ands_implicit(Expr *clause) { diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 816a2b2a576..7a92799700e 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -5455,6 +5455,8 @@ order_qual_clauses(PlannerInfo *root, List *clauses) static void copy_generic_path_info(Plan *dest, Path *src) { + ListCell *lc; + dest->disabled_nodes = src->disabled_nodes; dest->startup_cost = src->startup_cost; dest->total_cost = src->total_cost; @@ -5462,6 +5464,21 @@ copy_generic_path_info(Plan *dest, Path *src) dest->plan_width = src->pathtarget->width; dest->parallel_aware = src->parallel_aware; dest->parallel_safe = src->parallel_safe; + + /* Is this the right place to use makeNode()? */ + dest->app_extstats = makeNode(Applied_ExtStats); + dest->app_extstats->applied_stats = src->parent->applied_stats; + dest->app_extstats->applied_clauses_or = src->parent->applied_clauses_or; + dest->app_extstats->applied_clauses = NIL; + + foreach (lc, src->parent->applied_clauses) + { + List *clauses = (List *) lfirst(lc); + + dest->app_extstats->applied_clauses + = lappend(dest->app_extstats->applied_clauses, + maybe_extract_actual_clauses(clauses, false)); + } } /* diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index ff507331a06..b46ad85cba4 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -287,6 +287,10 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->partexprs = NULL; rel->nullable_partexprs = NULL; + rel->applied_stats = NIL; + rel->applied_clauses = NIL; + rel->applied_clauses_or = NIL; + /* * Pass assorted information down the inheritance hierarchy. */ @@ -769,6 +773,10 @@ build_join_rel(PlannerInfo *root, joinrel->partexprs = NULL; joinrel->nullable_partexprs = NULL; + joinrel->applied_stats = NIL; + joinrel->applied_clauses = NIL; + joinrel->applied_clauses_or = NIL; + /* Compute information relevant to the foreign relations. */ set_foreign_rel_properties(joinrel, outer_rel, inner_rel); @@ -953,6 +961,10 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->partexprs = NULL; joinrel->nullable_partexprs = NULL; + joinrel->applied_stats = NIL; + joinrel->applied_clauses = NIL; + joinrel->applied_clauses_or = NIL; + /* Compute information relevant to foreign relations. */ set_foreign_rel_properties(joinrel, outer_rel, inner_rel); diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index a80083d2323..b6cc5d7c2e2 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -499,6 +499,41 @@ extract_actual_clauses(List *restrictinfo_list, return result; } +/* + * maybe_extract_actual_clauses + * + * Just like extract_actual_clauses, but does not require the clauses to + * already be RestrictInfo. + * + * XXX Does not handle RestrictInfos nested in OR clauses. + */ +List * +maybe_extract_actual_clauses(List *restrictinfo_list, + bool pseudoconstant) +{ + List *result = NIL; + ListCell *l; + + foreach(l, restrictinfo_list) + { + RestrictInfo *rinfo; + Node *node = (Node *) lfirst(l); + + if (!IsA(node, RestrictInfo)) + { + result = lappend(result, node); + continue; + } + + rinfo = (RestrictInfo *) node; + + if (rinfo->pseudoconstant == pseudoconstant) + result = lappend(result, rinfo->clause); + } + + return result; +} + /* * extract_actual_join_clauses * diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index a8b63ec0884..c7367252aee 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -1854,6 +1854,14 @@ statext_mcv_clauselist_selectivity(PlannerInfo *root, List *clauses, int varReli list_exprs[listidx] = NULL; } + /* add it to the list of applied stats/clauses, if this flag is true */ + if (root->parse->isExplain_Stats) + { + rel->applied_stats = lappend(rel->applied_stats, stat); + rel->applied_clauses = lappend(rel->applied_clauses, stat_clauses); + rel->applied_clauses_or = lappend_int(rel->applied_clauses_or, (is_or) ? 1 : 0); + } + if (is_or) { bool *or_matches = NULL; diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index d3d1e485bb2..3258a3517a3 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -4083,6 +4083,7 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, ListCell *lc2; Bitmapset *matched = NULL; AttrNumber attnum_offset; + List *matched_exprs = NIL; /* * How much we need to offset the attnums? If there are no @@ -4130,6 +4131,9 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, matched = bms_add_member(matched, attnum); + /* track expressions matched by this statistics */ + matched_exprs = lappend(matched_exprs, varinfo->var); + found = true; } @@ -4158,6 +4162,9 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, matched = bms_add_member(matched, attnum); + /* track expressions matched by this statistics */ + matched_exprs = lappend(matched_exprs, expr); + /* there should be just one matching expression */ break; } @@ -4166,6 +4173,14 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, } } + /* add it to the list of applied stats/clauses, if this flag is true */ + if (root->parse->isExplain_Stats) + { + rel->applied_stats = lappend(rel->applied_stats, matched_info); + rel->applied_clauses = lappend(rel->applied_clauses, matched_exprs); + rel->applied_clauses_or = lappend_int(rel->applied_clauses_or, 2); /* 2: Use comma to deparse */ + } + /* Find the specific item that exactly matches the combination */ for (i = 0; i < stats->nitems; i++) { diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index bcfa5cb4add..36e3f72b041 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -36,6 +36,7 @@ #include "catalog/pg_publication.h" #include "catalog/pg_range.h" #include "catalog/pg_statistic.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_transform.h" #include "catalog/pg_type.h" @@ -3741,3 +3742,51 @@ get_subscription_name(Oid subid, bool missing_ok) return subname; } + +/* + * get_statistics_name + * Returns the name of a given extended statistics + * + * Returns a palloc'd copy of the string, or NULL if no such name. + */ +char * +get_statistics_name(Oid stxid) +{ + HeapTuple tp; + + tp = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(stxid)); + if (HeapTupleIsValid(tp)) + { + Form_pg_statistic_ext stxtup = (Form_pg_statistic_ext) GETSTRUCT(tp); + char *result; + + result = pstrdup(NameStr(stxtup->stxname)); + ReleaseSysCache(tp); + return result; + } + else + return NULL; +} + +/* + * get_statistics_namespace + * Returns the namespace OID of a given extended statistics + */ +Oid +get_statistics_namespace(Oid stxid) +{ + HeapTuple tp; + + tp = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(stxid)); + if (HeapTupleIsValid(tp)) + { + Form_pg_statistic_ext stxtup = (Form_pg_statistic_ext) GETSTRUCT(tp); + Oid result; + + result = stxtup->stxnamespace; + ReleaseSysCache(tp); + return result; + } + else + return InvalidOid; +} diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index ea7419951f4..797075126a0 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -55,6 +55,7 @@ typedef struct ExplainState bool memory; /* print planner's memory usage information */ bool settings; /* print modified settings */ bool generic; /* generate a generic plan */ + bool stats; /* print applied extended stats */ ExplainSerializeOption serialize; /* serialize the query's output? */ ExplainFormat format; /* output format */ /* state for output formatting --- not reset for each new plan tree */ diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h index 5473ce9a288..b32aa762c9b 100644 --- a/src/include/nodes/makefuncs.h +++ b/src/include/nodes/makefuncs.h @@ -95,6 +95,8 @@ extern Node *make_and_qual(Node *qual1, Node *qual2); extern Expr *make_ands_explicit(List *andclauses); extern List *make_ands_implicit(Expr *clause); +extern Expr *make_ors_explicit(List *orclauses); + extern IndexInfo *makeIndexInfo(int numattrs, int numkeyattrs, Oid amoid, List *expressions, List *predicates, bool unique, bool nulls_not_distinct, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 8dd421fa0ef..27b48629e66 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -249,6 +249,9 @@ typedef struct Query ParseLoc stmt_location; /* length in bytes; 0 means "rest of string" */ ParseLoc stmt_len pg_node_attr(query_jumble_ignore); + + /* if true, query is explain with stats option */ + bool isExplain_Stats; } Query; diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 00c700cc3e7..569d876fdfe 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -1068,6 +1068,11 @@ typedef struct RelOptInfo List **partexprs pg_node_attr(read_write_ignore); /* Nullable partition key expressions */ List **nullable_partexprs pg_node_attr(read_write_ignore); + + /* info about applied extended statistics */ + List *applied_stats; /* list of StatisticExtInfo */ + List *applied_clauses; /* list of lists of clauses */ + List *applied_clauses_or; /* are the clauses AND, OR, or Comma */ } RelOptInfo; /* diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 2a2cf816cb6..39d2d131348 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -177,6 +177,9 @@ typedef struct Plan */ Bitmapset *extParam; Bitmapset *allParam; + + /* info about applied extended statistics */ + struct Applied_ExtStats *app_extstats; } Plan; /* ---------------- @@ -1617,4 +1620,16 @@ typedef enum MonotonicFunction MONOTONICFUNC_BOTH = MONOTONICFUNC_INCREASING | MONOTONICFUNC_DECREASING, } MonotonicFunction; +/* + * Applied_ExtStats - Information to show applied Extend Statistics + * + */ +typedef struct Applied_ExtStats +{ + NodeTag type; + List *applied_stats; + List *applied_clauses; + List *applied_clauses_or; +} Applied_ExtStats; + #endif /* PLANNODES_H */ diff --git a/src/include/optimizer/restrictinfo.h b/src/include/optimizer/restrictinfo.h index ec91fc9c583..0c34af43138 100644 --- a/src/include/optimizer/restrictinfo.h +++ b/src/include/optimizer/restrictinfo.h @@ -50,6 +50,8 @@ extern bool restriction_is_securely_promotable(RestrictInfo *restrictinfo, extern List *get_actual_clauses(List *restrictinfo_list); extern List *extract_actual_clauses(List *restrictinfo_list, bool pseudoconstant); +extern List *maybe_extract_actual_clauses(List *restrictinfo_list, + bool pseudoconstant); extern void extract_actual_join_clauses(List *restrictinfo_list, Relids joinrelids, List **joinquals, diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index 6fab7aa6009..f1e73e389c8 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -207,6 +207,9 @@ extern char *get_publication_name(Oid pubid, bool missing_ok); extern Oid get_subscription_oid(const char *subname, bool missing_ok); extern char *get_subscription_name(Oid subid, bool missing_ok); +extern char *get_statistics_name(Oid stxid); +extern Oid get_statistics_namespace(Oid stxid); + #define type_is_array(typid) (get_element_type(typid) != InvalidOid) /* type_is_array_domain accepts both plain arrays and domains over arrays */ #define type_is_array_domain(typid) (get_base_element_type(typid) != InvalidOid) -- 2.43.5