This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f5390997022 HIVE-30145: SharedWorkOptimizer: decompose a discarded
scan after its semijoin filters are replaced (#6860)
f5390997022 is described below
commit f5390997022eb96826d18bb345e22a28181b3926
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Mon Oct 12 18:48:12 2026 +0200
HIVE-30145: SharedWorkOptimizer: decompose a discarded scan after its
semijoin filters are replaced (#6860)
---
.../hive/ql/optimizer/SharedWorkOptimizer.java | 3 +-
.../clientpositive/sharedwork_semijoin_ts_merge.q | 29 ++
.../llap/sharedwork_semijoin_ts_merge.q.out | 293 +++++++++++++++++++++
3 files changed, 324 insertions(+), 1 deletion(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
index bbfbc9e2b69..87fd12f28ec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -501,7 +501,6 @@ public boolean sharedWorkOptimization(ParseContext pctx,
SharedWorkOptimizerCach
}
DecomposedTs modelR = new DecomposedTs(retainableTsOp);
- DecomposedTs modelD = new DecomposedTs(discardableTsOp);
// Push filter on top of children for retainable
pushFilterToTopOfTableScan(optimizerCache, modelR);
@@ -510,6 +509,8 @@ public boolean sharedWorkOptimization(ParseContext pctx,
SharedWorkOptimizerCach
// For RemoveSemiJoin; this will clear the discardable's semijoin
filters
replaceSemijoinExpressions(discardableTsOp,
modelR.getSemiJoinFilter());
}
+ // Decompose after the replacement: the discardable's own semijoin
branches are removed below
+ DecomposedTs modelD = new DecomposedTs(discardableTsOp);
modelD.replaceTabAlias(discardableTsOp.getConf().getAlias(),
retainableTsOp.getConf().getAlias());
diff --git a/ql/src/test/queries/clientpositive/sharedwork_semijoin_ts_merge.q
b/ql/src/test/queries/clientpositive/sharedwork_semijoin_ts_merge.q
new file mode 100644
index 00000000000..eb56368eea0
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/sharedwork_semijoin_ts_merge.q
@@ -0,0 +1,29 @@
+--! qt:dataset:
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.auto.convert.join=false;
+set hive.tez.dynamic.semijoin.reduction=true;
+set hive.tez.bigtable.minsize.semijoin.reduction=1;
+set hive.tez.min.bloom.filter.entries=1;
+set hive.tez.dynamic.semijoin.reduction.threshold=-1;
+set hive.optimize.shared.work=true;
+set hive.optimize.shared.work.semijoin=false;
+
+create table fact_n1 (k bigint, v int);
+create table dim_n1 (k bigint, x int);
+insert into fact_n1 values (1, 10), (1, 20), (2, 30), (3, 40), (4, 1);
+insert into dim_n1 values (1, 1), (2, 1), (3, 2), (4, 1);
+analyze table fact_n1 compute statistics for columns;
+analyze table dim_n1 compute statistics for columns;
+
+-- Both scans of fact_n1 carry a semijoin filter from identical dim_n1
branches, but their
+-- filters differ, so the shared work optimizer merges only the scans and
keeps one semijoin
+-- branch: the discarded scan's filter must not keep referencing the removed
branch
+explain
+select a.s, b.s from
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1)
a,
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1
and f.v > 5) b;
+
+select a.s, b.s from
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1)
a,
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1
and f.v > 5) b;
diff --git
a/ql/src/test/results/clientpositive/llap/sharedwork_semijoin_ts_merge.q.out
b/ql/src/test/results/clientpositive/llap/sharedwork_semijoin_ts_merge.q.out
new file mode 100644
index 00000000000..fd4f9aae2f7
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/sharedwork_semijoin_ts_merge.q.out
@@ -0,0 +1,293 @@
+PREHOOK: query: create table fact_n1 (k bigint, v int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@fact_n1
+POSTHOOK: query: create table fact_n1 (k bigint, v int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@fact_n1
+PREHOOK: query: create table dim_n1 (k bigint, x int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dim_n1
+POSTHOOK: query: create table dim_n1 (k bigint, x int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dim_n1
+PREHOOK: query: insert into fact_n1 values (1, 10), (1, 20), (2, 30), (3, 40),
(4, 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@fact_n1
+POSTHOOK: query: insert into fact_n1 values (1, 10), (1, 20), (2, 30), (3,
40), (4, 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@fact_n1
+POSTHOOK: Lineage: fact_n1.k SCRIPT []
+POSTHOOK: Lineage: fact_n1.v SCRIPT []
+PREHOOK: query: insert into dim_n1 values (1, 1), (2, 1), (3, 2), (4, 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@dim_n1
+POSTHOOK: query: insert into dim_n1 values (1, 1), (2, 1), (3, 2), (4, 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@dim_n1
+POSTHOOK: Lineage: dim_n1.k SCRIPT []
+POSTHOOK: Lineage: dim_n1.x SCRIPT []
+PREHOOK: query: analyze table fact_n1 compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@fact_n1
+PREHOOK: Output: default@fact_n1
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table fact_n1 compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@fact_n1
+POSTHOOK: Output: default@fact_n1
+#### A masked pattern was here ####
+PREHOOK: query: analyze table dim_n1 compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@dim_n1
+PREHOOK: Output: default@dim_n1
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table dim_n1 compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@dim_n1
+POSTHOOK: Output: default@dim_n1
+#### A masked pattern was here ####
+Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1]] in Stage
'Reducer 4' is a cross product
+PREHOOK: query: explain
+select a.s, b.s from
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1)
a,
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1
and f.v > 5) b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dim_n1
+PREHOOK: Input: default@fact_n1
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select a.s, b.s from
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1)
a,
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1
and f.v > 5) b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dim_n1
+POSTHOOK: Input: default@fact_n1
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Reducer 8 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (XPROD_EDGE), Reducer 6 (XPROD_EDGE)
+ Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+ Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
+ Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: f
+ filterExpr: ((k is not null or ((v > 5) and k is not null))
and k BETWEEN DynamicValue(RS_7_d_k_min) AND DynamicValue(RS_7_d_k_max) and
in_bloom_filter(k, DynamicValue(RS_7_d_k_bloom_filter))) (type: boolean)
+ Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE
Column stats: COMPLETE
+ Filter Operator
+ predicate: (k is not null and k BETWEEN
DynamicValue(RS_7_d_k_min) AND DynamicValue(RS_7_d_k_max) and
in_bloom_filter(k, DynamicValue(RS_7_d_k_bloom_filter))) (type: boolean)
+ Statistics: Num rows: 5 Data size: 60 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: k (type: bigint), v (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 5 Data size: 60 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: bigint)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: bigint)
+ Statistics: Num rows: 5 Data size: 60 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: int)
+ Filter Operator
+ predicate: ((v > 5) and k is not null and k BETWEEN
DynamicValue(RS_7_d_k_min) AND DynamicValue(RS_7_d_k_max) and
in_bloom_filter(k, DynamicValue(RS_7_d_k_bloom_filter))) (type: boolean)
+ Statistics: Num rows: 4 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: k (type: bigint), v (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 4 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: bigint)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: bigint)
+ Statistics: Num rows: 4 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: d
+ filterExpr: ((x = 1) and k is not null) (type: boolean)
+ Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE
Column stats: COMPLETE
+ Filter Operator
+ predicate: ((x = 1) and k is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 24 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: k (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 16 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: bigint)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: bigint)
+ Statistics: Num rows: 2 Data size: 16 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: bigint)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: bigint)
+ Statistics: Num rows: 2 Data size: 16 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 16 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(_col0), max(_col0),
bloom_filter(_col0, expectedEntries=2)
+ minReductionHashAggr: 0.5
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 160 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 160 Basic
stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint), _col1
(type: bigint), _col2 (type: binary)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: bigint)
+ 1 _col0 (type: bigint)
+ outputColumnNames: _col1
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE
Column stats: COMPLETE
+ Group By Operator
+ aggregations: sum(_col1)
+ minReductionHashAggr: 0.6666666
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: bigint)
+ 1 _col0 (type: bigint)
+ outputColumnNames: _col2
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Group By Operator
+ aggregations: sum(_col2)
+ minReductionHashAggr: 0.5
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 6
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 8
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
bloom_filter(VALUE._col2, 1, expectedEntries=2)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE
Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE
Column stats: COMPLETE
+ value expressions: _col0 (type: bigint), _col1 (type:
bigint), _col2 (type: binary)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1]] in Stage
'Reducer 4' is a cross product
+PREHOOK: query: select a.s, b.s from
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1)
a,
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1
and f.v > 5) b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dim_n1
+PREHOOK: Input: default@fact_n1
+#### A masked pattern was here ####
+POSTHOOK: query: select a.s, b.s from
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1)
a,
+ (select sum(f.v) s from fact_n1 f join dim_n1 d on f.k = d.k where d.x = 1
and f.v > 5) b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dim_n1
+POSTHOOK: Input: default@fact_n1
+#### A masked pattern was here ####
+61 60
