From bd91aa6dd4a5ecac7de3106fc85bf2ff8787d76a Mon Sep 17 00:00:00 2001 From: Xiangxin Zeng Date: Thu, 15 Oct 2026 23:20:00 +0800 Subject: [PATCH 1/2] Estimate UNION ALL parent ndistinct ends When a simple UNION ALL is flattened into an appendrel parent, its output columns have no relation of their own and therefore no pg_statistic entry. Most distinct-value questions about those columns currently fall back to DEFAULT_NUM_DISTINCT, no matter how large the member relations are. Derive two member-based planning ends lazily, only when a planner consumer asks about a parent column. The low end is the largest member estimate. The high end is the sum of all member estimates, or zero when a member is not estimable. Most get_variable_numdistinct() callers use the result as a divisor, so this patch returns the low end there. That biases equality and join estimates away from understating selectivity. A later patch will handle consumers that need the count as a group count or numerator. Signed-off-by: Xiangxin Zeng --- src/backend/optimizer/path/allpaths.c | 128 +++++++++++++++++++++++++ src/backend/utils/adt/selfuncs.c | 30 ++++++ src/include/nodes/pathnodes.h | 11 +++ src/include/optimizer/paths.h | 1 + src/include/utils/selfuncs.h | 12 +++ src/test/regress/expected/union.out | 133 ++++++++++++++++++++++---- src/test/regress/sql/union.sql | 60 ++++++++++++ 7 files changed, 359 insertions(+), 16 deletions(-) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 24a6a8d11d..4461b60b9b 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -1375,6 +1375,134 @@ set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, add_paths_to_append_rel(root, rel, live_childrels); } +/* + * set_append_rel_ndistinct + * Compute per-column distinct-value estimates for an appendrel parent + * synthesized by flattening a UNION ALL. + * + * Such a parent has no relation of its own, so it has no pg_statistic entry and + * no route to one; without this, most distinct-value questions about its output + * columns would fall back to DEFAULT_NUM_DISTINCT (200), however much data the + * union actually covers. + * + * The true count depends on how much the members' value sets overlap. We + * therefore derive two planning ends: the largest member estimate and, when + * every non-dummy member is estimable, the sum of all member estimates. The + * ends inherit the members' estimation errors, so they are not mathematical + * bounds; consumers choose whichever end is conservative for their use. + * + * We compute the whole rel on first request and cache it on the RelOptInfo, + * rather than doing it for every appendrel in set_append_rel_size, so that we + * do not pay for columns nobody asks about. Callers must therefore check for + * NULL before assuming the arrays exist. + */ +void +set_append_rel_ndistinct(PlannerInfo *root, RelOptInfo *rel) +{ + RangeTblEntry *rte = root->simple_rte_array[rel->relid]; + int nattrs = rel->max_attr - rel->min_attr + 1; + MemoryContext oldcontext; + AttrNumber attno; + + Assert(rel->attr_ndistinct_lo == NULL && rel->attr_ndistinct_hi == NULL); + + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + rel->attr_ndistinct_lo = palloc0_array(double, nattrs); + rel->attr_ndistinct_hi = palloc0_array(double, nattrs); + MemoryContextSwitchTo(oldcontext); + + /* + * RTE_SUBQUERY + inh identifies an appendrel parent synthesized from a + * UNION ALL. Relation parents use inherited statistics instead. As with + * an ordinary subquery, do not look through a security barrier. + */ + if (rte->rtekind != RTE_SUBQUERY || !rte->inh || rte->security_barrier) + return; + + for (attno = rel->min_attr; attno <= rel->max_attr; attno++) + { + int pndx = attno - rel->min_attr; + List *childexprs = list_make1(NULL); + ListCell *lc; + double total = 0.0; + double largest = 0.0; + bool complete = true; + + if (attno <= 0) + continue; /* no members for system columns */ + + foreach(lc, root->append_rel_list) + { + AppendRelInfo *appinfo = (AppendRelInfo *) lfirst(lc); + RelOptInfo *childrel; + Node *childexpr; + EstimationInfo estinfo; + double childgroups; + + if (appinfo->parent_relid != rel->relid) + continue; + + childrel = root->simple_rel_array[appinfo->child_relid]; + if (childrel == NULL) + { + /* Lazy estimation can run before this member rel is built. */ + complete = false; + continue; + } + + /* Dummy members cannot contribute values to the union. */ + if (IS_DUMMY_REL(childrel)) + continue; + + /* + * Map the parent output column directly to the member expression. + * Do not use adjust_appendrel_attrs() here: for a non-Var member + * expression, it cannot propagate the parent's nullingrels, and + * those relids do not affect the set of output values. + */ + childexpr = (Node *) list_nth(appinfo->translated_vars, attno - 1); + + /* estimate_num_groups() expects Vars to identify an RTE */ + if (IsA(childexpr, Var) && ((Var *) childexpr)->varno == 0) + { + complete = false; + continue; + } + + /* + * Pass the member's post-qualification row count: the count we + * want is the distinct values this member contributes to the + * union's output, not the raw table's. + */ + linitial(childexprs) = childexpr; + childgroups = estimate_num_groups(root, childexprs, + childrel->rows, + NULL, &estinfo); + + /* + * A missing member estimate makes the sum incomplete. Continue + * scanning so that the low end can still use every estimable + * member, independent of member order. + */ + if (estinfo.flags & SELFLAG_USED_DEFAULT) + { + complete = false; + continue; + } + + total += childgroups; + largest = Max(largest, childgroups); + } + + /* + * The largest successful member estimate remains useful without the + * high end. Leave the high end zero to mark it unavailable. + */ + rel->attr_ndistinct_lo[pndx] = largest; + rel->attr_ndistinct_hi[pndx] = complete ? total : 0.0; + } +} + /* * set_grouped_rel_pathlist * If a grouped relation for the given 'rel' exists, build partial diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index e27ec9e5c2..69783c82e3 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -6303,6 +6303,27 @@ examine_simple_variable(PlannerInfo *root, Var *var, adjust_statstuple_for_grouping(subroot, var, vardata); } } + else if (rte->rtekind == RTE_SUBQUERY && rte->inh) + { + /* + * A flattened UNION ALL parent has no relation of its own and no + * pg_statistic entry. Lazily derive distinct-value ends from its + * members; zero means that the requested end is unavailable. + */ + RelOptInfo *rel = vardata->rel; + int pndx; + + if (rel->attr_ndistinct_lo == NULL) + set_append_rel_ndistinct(root, rel); + + if (var->varattno <= 0 || + var->varattno > rel->max_attr) + return; + + pndx = var->varattno - rel->min_attr; + vardata->ndistinct_lo = rel->attr_ndistinct_lo[pndx]; + vardata->ndistinct_hi = rel->attr_ndistinct_hi[pndx]; + } else { /* @@ -6719,6 +6740,15 @@ get_variable_numdistinct(VariableStatData *vardata, bool *isdefault) *isdefault = false; + /* + * This is currently used for flattened UNION ALL parents. The low end is + * conservative for callers that use a distinct count as a divisor; + * callers that need a numerator or a group count override it with the + * high end. Proven uniqueness still takes precedence. + */ + if (vardata->ndistinct_lo > 0.0 && !vardata->isunique) + return clamp_row_est(vardata->ndistinct_lo); + /* * Determine the stadistinct value to use. There are cases where we can * get an estimate even without a pg_statistic entry, or can get a better diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index c48e656ce8..77c40330c6 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -1079,6 +1079,17 @@ typedef struct RelOptInfo Relids *attr_needed pg_node_attr(read_write_ignore); /* array indexed [min_attr .. max_attr] */ int32 *attr_widths pg_node_attr(read_write_ignore); + + /* + * For a flattened UNION ALL parent, per-column distinct-value ends + * derived from member estimates: the low array holds the largest member + * estimate, and the high array holds their sum when every member is + * estimable. Zero means unavailable. Both arrays are indexed [min_attr + * .. max_attr] like attr_widths and are NULL until + * set_append_rel_ndistinct() runs. + */ + double *attr_ndistinct_lo pg_node_attr(read_write_ignore); + double *attr_ndistinct_hi pg_node_attr(read_write_ignore); /* zero-based set containing attnums of NOT NULL columns */ Bitmapset *notnullattnums; /* relids of outer joins that can null this baserel */ diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 051e38c118..dbfd55c3f7 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -283,5 +283,6 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, CompareType cmptype, bool nulls_first); extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +extern void set_append_rel_ndistinct(PlannerInfo *root, RelOptInfo *rel); #endif /* PATHS_H */ diff --git a/src/include/utils/selfuncs.h b/src/include/utils/selfuncs.h index 8d9fff95a1..bbcdede339 100644 --- a/src/include/utils/selfuncs.h +++ b/src/include/utils/selfuncs.h @@ -98,6 +98,18 @@ typedef struct VariableStatData * clause */ bool acl_ok; /* true if user has SELECT privilege on all * rows from the table or column */ + + /* + * Distinct-value planning ends derived without pg_statistic; zero if + * unavailable. For a flattened UNION ALL, the low end is the largest + * member estimate, and the high end is their sum when every member is + * estimable. Divisor-based estimates use the low end; group-count and + * numerator-based estimates use the high end when available. The ends do + * not describe value frequencies and must not replace an MCV or + * histogram. + */ + double ndistinct_lo; + double ndistinct_hi; } VariableStatData; #define ReleaseVariableStats(vardata) \ diff --git a/src/test/regress/expected/union.out b/src/test/regress/expected/union.out index 84abcd6b14..6ce430de99 100644 --- a/src/test/regress/expected/union.out +++ b/src/test/regress/expected/union.out @@ -1506,24 +1506,22 @@ select distinct q1 from union all select distinct * from int8_tbl i82) ss where q2 = q2; - QUERY PLAN ----------------------------------------------------------- - Unique - -> Merge Append - Sort Key: unnamed_subquery.q1 + QUERY PLAN +---------------------------------------------------- + HashAggregate + Group Key: unnamed_subquery.q1 + -> Append -> Subquery Scan on unnamed_subquery - -> Unique - -> Sort - Sort Key: i81.q1, i81.q2 - -> Seq Scan on int8_tbl i81 - Filter: (q2 IS NOT NULL) + -> HashAggregate + Group Key: i81.q1, i81.q2 + -> Seq Scan on int8_tbl i81 + Filter: (q2 IS NOT NULL) -> Subquery Scan on unnamed_subquery_1 - -> Unique - -> Sort - Sort Key: i82.q1, i82.q2 - -> Seq Scan on int8_tbl i82 - Filter: (q2 IS NOT NULL) -(15 rows) + -> HashAggregate + Group Key: i82.q1, i82.q2 + -> Seq Scan on int8_tbl i82 + Filter: (q2 IS NOT NULL) +(13 rows) select distinct q1 from (select distinct * from int8_tbl i81 @@ -1689,6 +1687,109 @@ select null::int[] union all select null::int[] union all select null::bigint[]; Output: NULL::bigint[] (10 rows) +-- +-- Test estimation of the distinct-value count of an appendrel parent that was +-- synthesized by flattening a UNION ALL. +-- +-- Such a parent has no relation of its own, hence no pg_statistic entry, so +-- before this was handled most questions about its columns' distinct-value +-- counts fell back to DEFAULT_NUM_DISTINCT (200), however much data the members +-- actually held. We now derive low and high member-based estimates and let +-- each consumer use the end that is conservative for its estimate. +-- +-- Restore the default plan choices first; the tests above turned several off. +reset enable_hashagg; +reset enable_sort; +reset enable_seqscan; +reset enable_indexscan; +reset enable_bitmapscan; +reset enable_indexonlyscan; +set max_parallel_workers_per_gather = 0; +create temp table ua1 (id int); +create temp table ua2 (id int); +insert into ua1 select g from generate_series(1, 2000) g; +insert into ua2 select g + 100000 from generate_series(1, 2000) g; +create temp table ud (id int primary key); +insert into ud select g from generate_series(1, 2000) g; +create temp table uu (id int); +insert into uu select g from generate_series(1, 2000) g; +analyze ua1; +analyze ua2; +analyze ud; +-- uu is deliberately left unanalyzed +-- A semi-join over the flattened union. With the historical default of 200 +-- distinct values, the inner side would look small enough to de-duplicate and +-- hash rather than being joined with semi-join semantics. +explain (costs off) +select count(*) from ud where ud.id in + (select id from ua1 union all select id from ua2); + QUERY PLAN +------------------------------------- + Aggregate + -> Hash Right Semi Join + Hash Cond: (ua1.id = ud.id) + -> Append + -> Seq Scan on ua1 + -> Seq Scan on ua2 + -> Hash + -> Seq Scan on ud +(8 rows) + +-- The union's group count feeds the join below, so it has to be derived from +-- the members rather than from a constant. +explain (costs off) +select count(*) from ud + join (select distinct id from + (select id from ua1 union all select id from ua2) x) y + on ud.id = y.id; + QUERY PLAN +----------------------------------------------- + Aggregate + -> Hash Join + Hash Cond: (ud.id = ua1.id) + -> Seq Scan on ud + -> Hash + -> HashAggregate + Group Key: ua1.id + -> Append + -> Seq Scan on ua1 + -> Seq Scan on ua2 +(10 rows) + +-- A member with no statistics cannot contribute to the sum, so the union has +-- no high end and consumers fall back to the largest member estimate. The +-- result must not depend on member order. +explain (costs off) +select count(*) from ud where ud.id in + (select id from ua1 union all select id from uu); + QUERY PLAN +------------------------------------- + Aggregate + -> Hash Right Semi Join + Hash Cond: (ua1.id = ud.id) + -> Append + -> Seq Scan on ua1 + -> Seq Scan on uu + -> Hash + -> Seq Scan on ud +(8 rows) + +explain (costs off) +select count(*) from ud where ud.id in + (select id from uu union all select id from ua1); + QUERY PLAN +------------------------------------ + Aggregate + -> Hash Right Semi Join + Hash Cond: (uu.id = ud.id) + -> Append + -> Seq Scan on uu + -> Seq Scan on ua1 + -> Hash + -> Seq Scan on ud +(8 rows) + +drop table ua1, ua2, ud, uu; -- Estimate a UNION's output rows as the sum of its children's distinct-group -- estimates, not the total input size. Ensure we get an index nested loop explain (costs off) diff --git a/src/test/regress/sql/union.sql b/src/test/regress/sql/union.sql index c8de276c2b..355c09863f 100644 --- a/src/test/regress/sql/union.sql +++ b/src/test/regress/sql/union.sql @@ -669,6 +669,66 @@ on true limit 1; explain (verbose, costs off) select null::int[] union all select null::int[] union all select null::bigint[]; +-- +-- Test estimation of the distinct-value count of an appendrel parent that was +-- synthesized by flattening a UNION ALL. +-- +-- Such a parent has no relation of its own, hence no pg_statistic entry, so +-- before this was handled most questions about its columns' distinct-value +-- counts fell back to DEFAULT_NUM_DISTINCT (200), however much data the members +-- actually held. We now derive low and high member-based estimates and let +-- each consumer use the end that is conservative for its estimate. +-- +-- Restore the default plan choices first; the tests above turned several off. +reset enable_hashagg; +reset enable_sort; +reset enable_seqscan; +reset enable_indexscan; +reset enable_bitmapscan; +reset enable_indexonlyscan; +set max_parallel_workers_per_gather = 0; + +create temp table ua1 (id int); +create temp table ua2 (id int); +insert into ua1 select g from generate_series(1, 2000) g; +insert into ua2 select g + 100000 from generate_series(1, 2000) g; +create temp table ud (id int primary key); +insert into ud select g from generate_series(1, 2000) g; +create temp table uu (id int); +insert into uu select g from generate_series(1, 2000) g; +analyze ua1; +analyze ua2; +analyze ud; +-- uu is deliberately left unanalyzed + +-- A semi-join over the flattened union. With the historical default of 200 +-- distinct values, the inner side would look small enough to de-duplicate and +-- hash rather than being joined with semi-join semantics. +explain (costs off) +select count(*) from ud where ud.id in + (select id from ua1 union all select id from ua2); + +-- The union's group count feeds the join below, so it has to be derived from +-- the members rather than from a constant. +explain (costs off) +select count(*) from ud + join (select distinct id from + (select id from ua1 union all select id from ua2) x) y + on ud.id = y.id; + +-- A member with no statistics cannot contribute to the sum, so the union has +-- no high end and consumers fall back to the largest member estimate. The +-- result must not depend on member order. +explain (costs off) +select count(*) from ud where ud.id in + (select id from ua1 union all select id from uu); + +explain (costs off) +select count(*) from ud where ud.id in + (select id from uu union all select id from ua1); + +drop table ua1, ua2, ud, uu; + -- Estimate a UNION's output rows as the sum of its children's distinct-group -- estimates, not the total input size. Ensure we get an index nested loop explain (costs off) -- 2.43.0