This is an automated email from the ASF dual-hosted git repository.

zhangbutao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new b058e3d00d0 HIVE-28363: Improve heuristics of FilterStatsRule without 
column stats (#5337)(Shohei Okumiya, reviewed by Zoltan Haindrich, Butao Zhang)
b058e3d00d0 is described below

commit b058e3d00d0f4315b55cb684bbff1944dd03bf8c
Author: okumin <[email protected]>
AuthorDate: Sat Sep 28 11:44:21 2024 +0900

    HIVE-28363: Improve heuristics of FilterStatsRule without column stats 
(#5337)(Shohei Okumiya, reviewed by Zoltan Haindrich, Butao Zhang)
---
 .../ql/optimizer/stats/annotation/StatsRulesProcFactory.java | 11 +++++++----
 ql/src/test/results/clientpositive/llap/explainuser_2.q.out  |  6 +++---
 .../test/results/clientpositive/llap/vector_between_in.q.out | 10 +++++-----
 .../results/clientpositive/llap/vectorized_timestamp.q.out   |  6 +++---
 .../results/clientpositive/perf/tpcds30tb/tez/query83.q.out  | 12 ++++++------
 5 files changed, 24 insertions(+), 21 deletions(-)

diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 79ca3f6220d..0c7fa176f6d 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -557,14 +557,17 @@ public class StatsRulesProcFactory {
       }
       for (int i = 0; i < columnStats.size(); i++) {
         long dvs = columnStats.get(i) == null ? 0 : 
columnStats.get(i).getCountDistint();
-        long intersectionSize = estimateIntersectionSize(aspCtx.getConf(), 
columnStats.get(i), values.get(i));
+        if (dvs == 0) {
+          factor *= 0.5;
+          continue;
+        }
         // (num of distinct vals for col in IN clause  / num of distinct vals 
for col )
-        double columnFactor = dvs == 0 ? 0.5d : (1.0d / dvs);
+        double columnFactor = 1.0 / dvs;
         if (!multiColumn) {
-          columnFactor *= intersectionSize;
+          columnFactor *= estimateIntersectionSize(aspCtx.getConf(), 
columnStats.get(i), values.get(i));
         }
         // max can be 1, even when ndv is larger in IN clause than in column 
stats
-        factor *= columnFactor > 1d ? 1d : columnFactor;
+        factor *= Math.min(columnFactor, 1.0);
       }
 
       // Clamp at 1 to be sure that we don't get out of range.
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out 
b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
index 00c18780bff..8599f31c4d4 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
@@ -1087,7 +1087,7 @@ Stage-0
                       <-Map 6 [BROADCAST_EDGE] vectorized, llap
                         SHUFFLE [RS_241]
                           PartitionCols:_col1, _col3
-                          Map Join Operator [MAPJOIN_239] (rows=550 width=10)
+                          Map Join Operator [MAPJOIN_239] (rows=275 width=10)
                             
Conds:RS_230._col0=SEL_237._col0(Inner),Output:["_col1","_col2","_col3"]
                           <-Map 1 [BROADCAST_EDGE] vectorized, llap
                             BROADCAST [RS_230]
@@ -1098,9 +1098,9 @@ Stage-0
                                   predicate:(v2 is not null and v3 is not null 
and k1 is not null)
                                   TableScan [TS_0] (rows=170 width=34)
                                     
default@cs,cs,Tbl:COMPLETE,Col:NONE,Output:["k1","v2","k3","v3"]
-                          <-Select Operator [SEL_237] (rows=500 width=10)
+                          <-Select Operator [SEL_237] (rows=250 width=10)
                               Output:["_col0"]
-                              Filter Operator [FIL_235] (rows=500 width=10)
+                              Filter Operator [FIL_235] (rows=250 width=10)
                                 predicate:((value) IN ('2000Q1', '2000Q2', 
'2000Q3') and key is not null)
                                 TableScan [TS_12] (rows=500 width=10)
                                   
default@src,d1,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
diff --git a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out 
b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
index 51f401fe018..2a8a77a9f49 100644
--- a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
                         native: true
                         predicateExpression: FilterLongColumnInList(col 
3:date, values [-67, -171])
                     predicate: (cdate) IN (DATE'1969-10-26', DATE'1969-07-14') 
(type: boolean)
-                    Statistics: Num rows: 12289 Data size: 339304 Basic stats: 
COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 6145 Data size: 169680 Basic stats: 
COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: cdate (type: date)
                       outputColumnNames: _col0
@@ -70,7 +70,7 @@ STAGE PLANS:
                           className: VectorSelectOperator
                           native: true
                           projectedOutputColumnNums: [3]
-                      Statistics: Num rows: 12289 Data size: 339304 Basic 
stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 6145 Data size: 169680 Basic 
stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: date)
                         null sort order: z
@@ -79,7 +79,7 @@ STAGE PLANS:
                             className: VectorReduceSinkObjectHashOperator
                             native: true
                             nativeConditionsMet: 
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine 
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, 
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
-                        Statistics: Num rows: 12289 Data size: 339304 Basic 
stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 6145 Data size: 169680 Basic 
stats: COMPLETE Column stats: COMPLETE
             Execution mode: vectorized, llap
             LLAP IO: all inputs
             Map Vectorization:
@@ -107,13 +107,13 @@ STAGE PLANS:
                     className: VectorSelectOperator
                     native: true
                     projectedOutputColumnNums: [0]
-                Statistics: Num rows: 12289 Data size: 339304 Basic stats: 
COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 6145 Data size: 169680 Basic stats: 
COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   File Sink Vectorization:
                       className: VectorFileSinkOperator
                       native: false
-                  Statistics: Num rows: 12289 Data size: 339304 Basic stats: 
COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 6145 Data size: 169680 Basic stats: 
COMPLETE Column stats: COMPLETE
                   table:
                       input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
                       output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out 
b/ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out
index 3e1d638d594..8f722cdd70b 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_timestamp.q.out
@@ -290,7 +290,7 @@ STAGE PLANS:
                         native: true
                         predicateExpression: FilterTimestampColumnInList(col 
0:timestamp, values [0001-01-02 16:00:00.0, 0002-02-03 16:00:00.0])
                     predicate: (ts) IN (TIMESTAMP'0001-01-01 00:00:00', 
TIMESTAMP'0002-02-02 00:00:00') (type: boolean)
-                    Statistics: Num rows: 2 Data size: 80 Basic stats: 
COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 1 Data size: 40 Basic stats: 
COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: ts (type: timestamp)
                       outputColumnNames: _col0
@@ -298,13 +298,13 @@ STAGE PLANS:
                           className: VectorSelectOperator
                           native: true
                           projectedOutputColumnNums: [0]
-                      Statistics: Num rows: 2 Data size: 80 Basic stats: 
COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 1 Data size: 40 Basic stats: 
COMPLETE Column stats: COMPLETE
                       File Output Operator
                         compressed: false
                         File Sink Vectorization:
                             className: VectorFileSinkOperator
                             native: false
-                        Statistics: Num rows: 2 Data size: 80 Basic stats: 
COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 1 Data size: 40 Basic stats: 
COMPLETE Column stats: COMPLETE
                         table:
                             input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
                             output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git 
a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query83.q.out 
b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query83.q.out
index a1743163ec4..39b24847379 100644
--- a/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query83.q.out
+++ b/ql/src/test/results/clientpositive/perf/tpcds30tb/tez/query83.q.out
@@ -83,14 +83,14 @@ STAGE PLANS:
                   Statistics: Num rows: 73049 Data size: 4382940 Basic stats: 
COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: ((d_date) IN (DATE'1998-01-02', 
DATE'1998-10-15', DATE'1998-11-10') and d_week_seq is not null) (type: boolean)
-                    Statistics: Num rows: 73049 Data size: 4382940 Basic 
stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 36525 Data size: 2191500 Basic 
stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: d_week_seq (type: int)
                       outputColumnNames: _col0
-                      Statistics: Num rows: 73049 Data size: 292196 Basic 
stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 36525 Data size: 146100 Basic 
stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         keys: _col0 (type: int)
-                        minReductionHashAggr: 0.8453504
+                        minReductionHashAggr: 0.690705
                         mode: hash
                         outputColumnNames: _col0
                         Statistics: Num rows: 11297 Data size: 45188 Basic 
stats: COMPLETE Column stats: COMPLETE
@@ -325,14 +325,14 @@ STAGE PLANS:
                             Statistics: Num rows: 36524 Data size: 2045344 
Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: ((d_date) IN (DATE'1998-01-02', 
DATE'1998-10-15', DATE'1998-11-10') and d_week_seq is not null) (type: boolean)
-                    Statistics: Num rows: 73049 Data size: 4382940 Basic 
stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 36525 Data size: 2191500 Basic 
stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: d_week_seq (type: int)
                       outputColumnNames: _col0
-                      Statistics: Num rows: 73049 Data size: 292196 Basic 
stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 36525 Data size: 146100 Basic 
stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         keys: _col0 (type: int)
-                        minReductionHashAggr: 0.8453504
+                        minReductionHashAggr: 0.690705
                         mode: hash
                         outputColumnNames: _col0
                         Statistics: Num rows: 11297 Data size: 45188 Basic 
stats: COMPLETE Column stats: COMPLETE

Reply via email to