This is an automated email from the ASF dual-hosted git repository.
soumyakanti3578 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 89a6b5c67ec HIVE-26368: DISTINCT keyword from Count UDF is removed
from query plan when CBO is enabled
89a6b5c67ec is described below
commit 89a6b5c67ec8304ddccf996ed0ac64a576952195
Author: Soumyakanti Das <[email protected]>
AuthorDate: Tue Aug 11 11:21:57 2026 -0700
HIVE-26368: DISTINCT keyword from Count UDF is removed from query plan when
CBO is enabled
---
.../rules/HiveExpandDistinctAggregatesRule.java | 15 +-
.../clientpositive/count_distinct_multi_col.q | 70 ++++++
.../llap/count_distinct_multi_col.q.out | 238 +++++++++++++++++++++
3 files changed, 317 insertions(+), 6 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
index 3a46498298b..6dc4ccbd48c 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
@@ -161,7 +161,14 @@ public void onMatch(RelOptRuleCall call) {
// arguments then we can use a more efficient form.
final RelMetadataQuery mq = call.getMetadataQuery();
if ((nonDistinctCount == 0) && (argListSets.size() == 1)) {
- for (Integer arg : argListSets.iterator().next()) {
+ List<Integer> argList = argListSets.iterator().next();
+ // When COUNT(DISTINCT a, b, ...) has multiple columns convertMonopole
-> rewriteAggCalls
+ // would produce COUNT(a, b) without DISTINCT at the top level, which is
semantically
+ // wrong and is rejected by GenericUDAFCount.
+ if (argList.size() > 1 && numCountDistinct > 0) {
+ return;
+ }
+ for (Integer arg : argList) {
Set<RelColumnOrigin> colOrigs =
mq.getColumnOrigins(aggregate.getInput(), arg);
if (null != colOrigs) {
for (RelColumnOrigin colOrig : colOrigs) {
@@ -173,12 +180,8 @@ public void onMatch(RelOptRuleCall call) {
}
}
}
- RelNode converted =
- convertMonopole(
- aggregate,
- argListSets.iterator().next());
+ RelNode converted = convertMonopole(aggregate, argList);
call.transformTo(converted);
- return;
}
}
diff --git a/ql/src/test/queries/clientpositive/count_distinct_multi_col.q
b/ql/src/test/queries/clientpositive/count_distinct_multi_col.q
new file mode 100644
index 00000000000..edd20477fb1
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/count_distinct_multi_col.q
@@ -0,0 +1,70 @@
+create table t (col1 string, col2 int, col3 int, col4 date);
+
+insert into t values
+ ('a', 1, 2, '2022-01-01'),
+ ('a', 1, 2, '2022-01-01'),
+ ('a', 1, 2, '2022-01-02'),
+ ('a', 1, 22, '2022-01-02'),
+ ('a', 11, 2, '2022-01-01'),
+ ('a', 11, 2, '2022-01-02');
+
+-- single-column: HiveExpandDistinctAggregatesRule matched -> two
HiveAggregate operators in plan
+explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
+FROM t
+GROUP BY col1, col2;
+
+SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2;
+
+-- multi-column: HiveExpandDistinctAggregatesRule !matched -> single
HiveAggregate operator
+explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2;
+
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2;
+
+-- multi-column distinct alongside a non-distinct aggregate: single
HiveAggregate operator
+explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
+FROM t
+GROUP BY col1, col2;
+
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2;
+
+-- multiple COUNT DISTINCT with different single-column arg: uses grouping-sets
+explain cbo
+SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
+FROM t
+GROUP BY col1;
+
+SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
+FROM t
+GROUP BY col1;
+
+-- 2 COUNT DISTINCTs each with multiple columns: uses grouping-sets path
+explain cbo
+SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
+FROM t
+GROUP BY col1;
+
+SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
+FROM t
+GROUP BY col1;
+
+-- CTAS
+create table t_ctas as
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2;
+
+select * from t_ctas ORDER BY col1, col2;
diff --git
a/ql/src/test/results/clientpositive/llap/count_distinct_multi_col.q.out
b/ql/src/test/results/clientpositive/llap/count_distinct_multi_col.q.out
new file mode 100644
index 00000000000..034c1a9f3ac
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/count_distinct_multi_col.q.out
@@ -0,0 +1,238 @@
+PREHOOK: query: create table t (col1 string, col2 int, col3 int, col4 date)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t (col1 string, col2 int, col3 int, col4 date)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert into t values
+ ('a', 1, 2, '2022-01-01'),
+ ('a', 1, 2, '2022-01-01'),
+ ('a', 1, 2, '2022-01-02'),
+ ('a', 1, 22, '2022-01-02'),
+ ('a', 11, 2, '2022-01-01'),
+ ('a', 11, 2, '2022-01-02')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t values
+ ('a', 1, 2, '2022-01-01'),
+ ('a', 1, 2, '2022-01-01'),
+ ('a', 1, 2, '2022-01-02'),
+ ('a', 1, 22, '2022-01-02'),
+ ('a', 11, 2, '2022-01-01'),
+ ('a', 11, 2, '2022-01-02')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.col1 SCRIPT []
+POSTHOOK: Lineage: t.col2 SCRIPT []
+POSTHOOK: Lineage: t.col3 SCRIPT []
+POSTHOOK: Lineage: t.col4 SCRIPT []
+PREHOOK: query: explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
+FROM t
+GROUP BY col1, col2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
+FROM t
+GROUP BY col1, col2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(col1=[$0], col2=[$1], cnt=[$2])
+ HiveAggregate(group=[{0, 1}], agg#0=[count($2)])
+ HiveProject(col1=[$0], col2=[$1], col3=[$2])
+ HiveAggregate(group=[{0, 1, 2}])
+ HiveTableScan(table=[[default, t]], table:alias=[t])
+
+PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+a 1 2
+a 11 1
+PREHOOK: query: explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(col1=[$0], col2=[$1], cnt=[$2])
+ HiveAggregate(group=[{0, 1}], agg#0=[count(DISTINCT $2, $3)])
+ HiveTableScan(table=[[default, t]], table:alias=[t])
+
+PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+a 1 3
+a 11 2
+PREHOOK: query: explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
+FROM t
+GROUP BY col1, col2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
+FROM t
+GROUP BY col1, col2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(col1=[$0], col2=[$1], cnt=[$2], s=[$3])
+ HiveAggregate(group=[{0, 1}], agg#0=[count(DISTINCT $2, $3)],
agg#1=[sum($2)])
+ HiveTableScan(table=[[default, t]], table:alias=[t])
+
+PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt,
SUM(col3) AS s
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt,
SUM(col3) AS s
+FROM t
+GROUP BY col1, col2
+ORDER BY col1, col2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+a 1 3 28
+a 11 2 4
+PREHOOK: query: explain cbo
+SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
+FROM t
+GROUP BY col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
+FROM t
+GROUP BY col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(col1=[$0], c3=[$1], c2=[$2])
+ HiveAggregate(group=[{2}], agg#0=[count($0)], agg#1=[count($1)])
+ HiveProject($f0=[CASE(AND(=($3, 1), IS NOT NULL($1)), 1, null:INTEGER)],
$f1=[CASE(AND(=($3, 2), IS NOT NULL($2)), 1, null:INTEGER)], $f2=[$0])
+ HiveAggregate(group=[{0, 1, 2}], groups=[[{0, 1}, {0, 2}]],
GROUPING__ID=[GROUPING__ID()])
+ HiveProject($f0=[$0], $f1=[$2], $f2=[$1])
+ HiveTableScan(table=[[default, t]], table:alias=[t])
+
+PREHOOK: query: SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2)
AS c2
+FROM t
+GROUP BY col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2)
AS c2
+FROM t
+GROUP BY col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+a 2 2
+PREHOOK: query: explain cbo
+SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
+FROM t
+GROUP BY col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
+FROM t
+GROUP BY col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(col1=[$0], c1=[$1], c2=[$2])
+ HiveAggregate(group=[{2}], agg#0=[count($0)], agg#1=[count($1)])
+ HiveProject($f0=[CASE(=($4, 1), 1, null:INTEGER)], $f1=[CASE(=($4, 4), 1,
null:INTEGER)], $f2=[$0])
+ HiveAggregate(group=[{0, 1, 2, 3}], groups=[[{0, 1, 2}, {0, 2, 3}]],
GROUPING__ID=[GROUPING__ID()])
+ HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3])
+ HiveTableScan(table=[[default, t]], table:alias=[t])
+
+PREHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT
col3, col4) AS c2
+FROM t
+GROUP BY col1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT
col3, col4) AS c2
+FROM t
+GROUP BY col1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+a 3 3
+PREHOOK: query: create table t_ctas as
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@t
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t_ctas
+POSTHOOK: query: create table t_ctas as
+SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
+FROM t
+GROUP BY col1, col2
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@t
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t_ctas
+POSTHOOK: Lineage: t_ctas.cnt EXPRESSION [(t)t.FieldSchema(name:col3,
type:int, comment:null), (t)t.FieldSchema(name:col4, type:date, comment:null), ]
+POSTHOOK: Lineage: t_ctas.col1 SIMPLE [(t)t.FieldSchema(name:col1,
type:string, comment:null), ]
+POSTHOOK: Lineage: t_ctas.col2 SIMPLE [(t)t.FieldSchema(name:col2, type:int,
comment:null), ]
+PREHOOK: query: select * from t_ctas ORDER BY col1, col2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t_ctas
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t_ctas ORDER BY col1, col2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t_ctas
+#### A masked pattern was here ####
+a 1 3
+a 11 2
