This is an automated email from the ASF dual-hosted git repository.
zhangbutao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b058e3d00d0 HIVE-28363: Improve heuristics of FilterStatsRule without
column stats (#5337)(Shohei Okumiya, reviewed by Zoltan Haindrich, Butao Zhang)
b058e3d00d0 is described below
commit b058e3d00d0f4315b55cb684bbff1944dd03bf8c
Author: okumin <[email protected]>
AuthorDate: Sat Sep 28 11:44:21 2024 +0900
HIVE-28363: Improve heuristics of FilterStatsRule without column stats
(#5337)(Shohei Okumiya, reviewed by Zoltan Haindrich, Butao Zhang)
---
.../ql/optimizer/stats/annotation/StatsRulesProcFactory.java | 11 +++++++----
ql/src/test/results/clientpositive/llap/explainuser_2.q.out | 6 +++---
.../test/results/clientpositive/llap/vector_between_in.q.out | 10 +++++-----
.../results/clientpositive/llap/vectorized_timestamp.q.out | 6 +++---
.../results/clientpositive/perf/tpcds30tb/tez/query83.q.out | 12 ++++++------
5 files changed, 24 insertions(+), 21 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 79ca3f6220d..0c7fa176f6d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -557,14 +557,17 @@ public class StatsRulesProcFactory {
}
for (int i = 0; i < columnStats.size(); i++) {
long dvs = columnStats.get(i) == null ? 0 :
columnStats.get(i).getCountDistint();
- long intersectionSize = estimateIntersectionSize(aspCtx.getConf(),
columnStats.get(i), values.get(i));
+ if (dvs == 0) {
+ factor *= 0.5;
+ continue;
+ }
// (num of distinct vals for col in IN clause / num of distinct vals
for col )
- double columnFactor = dvs == 0 ? 0.5d : (1.0d / dvs);
+ double columnFactor = 1.0 / dvs;
if (!multiColumn) {
- columnFactor *= intersectionSize;
+ columnFactor *= estimateIntersectionSize(aspCtx.getConf(),
columnStats.get(i), values.get(i));
}
// max can be 1, even when ndv is larger in IN clause than in column
stats
- factor *= columnFactor > 1d ? 1d : columnFactor;
+ factor *= Math.min(columnFactor, 1.0);
}
// Clamp at 1 to be sure that we don't get out of range.
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
index 00c18780bff..8599f31c4d4 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
@@ -1087,7 +1087,7 @@ Stage-0
<-Map 6 [BROADCAST_EDGE] vectorized, llap
SHUFFLE [RS_241]
PartitionCols:_col1, _col3
- Map Join Operator [MAPJOIN_239] (rows=550 width=10)
+ Map Join Operator [MAPJOIN_239] (rows=275 width=10)
Conds:RS_230._col0=SEL_237._col0(Inner),Output:["_col1","_col2","_col3"]
<-Map 1 [BROADCAST_EDGE] vectorized, llap
BROADCAST [RS_230]
@@ -1098,9 +1098,9 @@ Stage-0
predicate:(v2 is not null and v3 is not null
and k1 is not null)
TableScan [TS_0] (rows=170 width=34)
default@cs,cs,Tbl:COMPLETE,Col:NONE,Output:["k1","v2","k3","v3"]
- <-Select Operator [SEL_237] (rows=500 width=10)
+ <-Select Operator [SEL_237] (rows=250 width=10)
Output:["_col0"]
- Filter Operator [FIL_235] (rows=500 width=10)
+ Filter Operator [FIL_235] (rows=250 width=10)
predicate:((value) IN ('2000Q1', '2000Q2',
'2000Q3') and key is not null)
TableScan [TS_12] (rows=500 width=10)
default@src,d1,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
diff --git a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
index 51f401fe018..2a8a77a9f49 100644
--- a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterLongColumnInList(col
3:date, values [-67, -171])
predicate: (cdate) IN (DATE'1969-10-26', DATE'1969-07-14')
(type: boolean)
- Statistics: Num rows: 12289 Data size: 339304 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6145 Data size: 169680 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: cdate (type: date)
outputColumnNames: _col0
@@ -70,7 +70,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [3]
- Statistics: Num rows: 12289 Data size: 339304 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6145 Data size: 169680 Basic
stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: date)
null sort order: z
@@ -79,7 +79,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12289 Data size: 339304 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6145 Data size: 169680 Basic
stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -107,13 +107,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0]
- Statistics: Num rows: 12289 Data size: 339304 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6145 Data size: 169680 Basic stats:
COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 12289 Data size: 339304 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6145 Data size: 169680 Basic stats:
COMPLETE Column stats: COMPLETE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out
b/ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out
index 3e1d638d594..8f722cdd70b 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out
@@ -290,7 +290,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterTimestampColumnInList(col
0:timestamp, values [0001-01-02 16:00:00.0, 0002-02-03 16:00:00.0])
predicate: (ts) IN (TIMESTAMP'0001-01-01 00:00:00',
TIMESTAMP'0002-02-02 00:00:00') (type: boolean)
- Statistics: Num rows: 2 Data size: 80 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 40 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
expressions: ts (type: timestamp)
outputColumnNames: _col0
@@ -298,13 +298,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0]
- Statistics: Num rows: 2 Data size: 80 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 40 Basic stats:
COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 2 Data size: 80 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 40 Basic stats:
COMPLETE Column stats: COMPLETE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git
a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query83.q.out
b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query83.q.out
index a1743163ec4..39b24847379 100644
--- a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query83.q.out
+++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query83.q.out
@@ -83,14 +83,14 @@ STAGE PLANS:
Statistics: Num rows: 73049 Data size: 4382940 Basic stats:
COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((d_date) IN (DATE'1998-01-02',
DATE'1998-10-15', DATE'1998-11-10') and d_week_seq is not null) (type: boolean)
- Statistics: Num rows: 73049 Data size: 4382940 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 36525 Data size: 2191500 Basic
stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: d_week_seq (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 73049 Data size: 292196 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 36525 Data size: 146100 Basic
stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int)
- minReductionHashAggr: 0.8453504
+ minReductionHashAggr: 0.690705
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 11297 Data size: 45188 Basic
stats: COMPLETE Column stats: COMPLETE
@@ -325,14 +325,14 @@ STAGE PLANS:
Statistics: Num rows: 36524 Data size: 2045344
Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((d_date) IN (DATE'1998-01-02',
DATE'1998-10-15', DATE'1998-11-10') and d_week_seq is not null) (type: boolean)
- Statistics: Num rows: 73049 Data size: 4382940 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 36525 Data size: 2191500 Basic
stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: d_week_seq (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 73049 Data size: 292196 Basic
stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 36525 Data size: 146100 Basic
stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int)
- minReductionHashAggr: 0.8453504
+ minReductionHashAggr: 0.690705
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 11297 Data size: 45188 Basic
stats: COMPLETE Column stats: COMPLETE