This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 43604db9d29 HIVE-30097: SharedWorkOptimizer never merges Iceberg reads
that carry dynamic partition pruning (#6824)
43604db9d29 is described below
commit 43604db9d29a85e0b847b3133451dfdcaffba624
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Fri Oct 2 20:27:54 2026 +0200
HIVE-30097: SharedWorkOptimizer never merges Iceberg reads that carry
dynamic partition pruning (#6824)
---
.../queries/positive/dynamic_partition_pruning.q | 18 ++
.../positive/dynamic_partition_pruning.q.out | 204 +++++++++++++++++++++
.../hive/ql/plan/DynamicPruningEventDesc.java | 7 +-
3 files changed, 223 insertions(+), 6 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_pruning.q
b/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_pruning.q
index 8b469e26dc5..dfb1843fe45 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_pruning.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_pruning.q
@@ -153,6 +153,24 @@ EXPLAIN select count(*) from srcpart_iceberg join
srcpart_date_hour_n0_iceberg o
select count(*) from srcpart_iceberg join srcpart_date_hour_n0_iceberg on
(srcpart_iceberg.ds = srcpart_date_hour_n0_iceberg.ds and srcpart_iceberg.hr =
srcpart_date_hour_n0_iceberg.hr) where srcpart_date_hour_n0_iceberg.hour = 11
and (srcpart_date_hour_n0_iceberg.`date` = '2008-04-08' or
srcpart_date_hour_n0_iceberg.`date` = '2008-04-09');
select count(*) from srcpart where (ds = '2008-04-08' or ds = '2008-04-09')
and hr = 11;
+-- the same pruned read twice: the shared work optimizer merges both reads,
their dynamic partition
+-- pruning event included, into one pipeline
+reset -d hive.tez.min.bloom.filter.entries
hive.tez.bigtable.minsize.semijoin.reduction;
+
+EXPLAIN with v as (
+ select srcpart_iceberg.key, count(*) c from srcpart_iceberg join
srcpart_date_n2 on (srcpart_iceberg.ds = srcpart_date_n2.ds)
+ where srcpart_date_n2.`date` = '2008-04-08'
+ group by srcpart_iceberg.key
+)
+select sum(v1.c * v2.c) from v v1 join v v2 on (v1.key = v2.key);
+
+with v as (
+ select srcpart_iceberg.key, count(*) c from srcpart_iceberg join
srcpart_date_n2 on (srcpart_iceberg.ds = srcpart_date_n2.ds)
+ where srcpart_date_n2.`date` = '2008-04-08'
+ group by srcpart_iceberg.key
+)
+select sum(v1.c * v2.c) from v v1 join v v2 on (v1.key = v2.key);
+
drop table srcpart_iceberg;
drop table srcpart_date_hour_n0_iceberg;
drop table srcpart_date_n2;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out
b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out
index 94c1e9cb2b8..962fc0b76fd 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out
@@ -4302,6 +4302,210 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
POSTHOOK: Output: hdfs://### HDFS PATH ###
1000
+PREHOOK: query: EXPLAIN with v as (
+ select srcpart_iceberg.key, count(*) c from srcpart_iceberg join
srcpart_date_n2 on (srcpart_iceberg.ds = srcpart_date_n2.ds)
+ where srcpart_date_n2.`date` = '2008-04-08'
+ group by srcpart_iceberg.key
+)
+select sum(v1.c * v2.c) from v v1 join v v2 on (v1.key = v2.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_date_n2
+PREHOOK: Input: default@srcpart_iceberg
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: EXPLAIN with v as (
+ select srcpart_iceberg.key, count(*) c from srcpart_iceberg join
srcpart_date_n2 on (srcpart_iceberg.ds = srcpart_date_n2.ds)
+ where srcpart_date_n2.`date` = '2008-04-08'
+ group by srcpart_iceberg.key
+)
+select sum(v1.c * v2.c) from v v1 join v v2 on (v1.key = v2.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_date_n2
+POSTHOOK: Input: default@srcpart_iceberg
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 5 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+ Reducer 4 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_iceberg
+ filterExpr: (ds is not null and key is not null) (type:
boolean)
+ Statistics: Num rows: 2000 Data size: 542000 Basic stats:
COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 542000 Basic stats:
COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2000 Data size: 542000 Basic
stats: COMPLETE Column stats: PARTIAL
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ input vertices:
+ 1 Map 5
+ Statistics: Num rows: 1000 Data size: 87000 Basic
stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ aggregations: count()
+ keys: _col0 (type: string)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 158 Data size: 15010 Basic
stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 158 Data size: 15010 Basic
stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 158 Data size: 15010 Basic
stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: bigint)
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date_n2
+ filterExpr: ((date = '2008-04-08') and ds is not null)
(type: boolean)
+ Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE
Column stats: COMPLETE
+ Filter Operator
+ predicate: ((date = '2008-04-08') and ds is not null)
(type: boolean)
+ Statistics: Num rows: 1 Data size: 188 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: ds (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 94 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 94 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 94 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 94 Basic stats:
COMPLETE Column stats: COMPLETE
+ Dynamic Partitioning Event Operator
+ Target column: ds (string)
+ Target Input: srcpart_iceberg
+ Partition key expr: ds
+ Statistics: Num rows: 1 Data size: 94 Basic stats:
COMPLETE Column stats: COMPLETE
+ Target Vertex: Map 1
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 158 Data size: 15010 Basic stats:
COMPLETE Column stats: PARTIAL
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col3
+ input vertices:
+ 1 Reducer 4
+ Statistics: Num rows: 158 Data size: 2528 Basic stats:
COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: (_col3 * _col1) (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 158 Data size: 2528 Basic stats:
COMPLETE Column stats: PARTIAL
+ Group By Operator
+ aggregations: sum(_col0)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: PARTIAL
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 158 Data size: 15010 Basic stats:
COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 158 Data size: 15010 Basic stats:
COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: bigint)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: with v as (
+ select srcpart_iceberg.key, count(*) c from srcpart_iceberg join
srcpart_date_n2 on (srcpart_iceberg.ds = srcpart_date_n2.ds)
+ where srcpart_date_n2.`date` = '2008-04-08'
+ group by srcpart_iceberg.key
+)
+select sum(v1.c * v2.c) from v v1 join v v2 on (v1.key = v2.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_date_n2
+PREHOOK: Input: default@srcpart_iceberg
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: with v as (
+ select srcpart_iceberg.key, count(*) c from srcpart_iceberg join
srcpart_date_n2 on (srcpart_iceberg.ds = srcpart_date_n2.ds)
+ where srcpart_date_n2.`date` = '2008-04-08'
+ group by srcpart_iceberg.key
+)
+select sum(v1.c * v2.c) from v v1 join v v2 on (v1.key = v2.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_date_n2
+POSTHOOK: Input: default@srcpart_iceberg
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+4112
PREHOOK: query: drop table srcpart_iceberg
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@srcpart_iceberg
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
index c4342990be9..7f51303087c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
@@ -58,10 +58,6 @@ public void setPredicate(ExprNodeDesc predicate) {
this.predicate = predicate;
}
- public String getPartPredicateString() {
- return this.predicate != null ? this.predicate.getExprString() : "-";
- }
-
public TableScanOperator getTableScan() {
return tableScan;
}
@@ -127,8 +123,7 @@ public boolean isSame(OperatorDesc other) {
DynamicPruningEventDesc otherDesc = (DynamicPruningEventDesc) other;
return Objects.equals(getTargetColumnName(),
otherDesc.getTargetColumnName()) &&
Objects.equals(getTargetColumnType(),
otherDesc.getTargetColumnType()) &&
- Objects.equals(getPartKeyString(), otherDesc.getPartKeyString()) &&
- Objects.equals(getPartPredicateString(),
otherDesc.getPartPredicateString());
+ Objects.equals(getPartKeyString(), otherDesc.getPartKeyString());
}
return false;
}
