HIVE-19338 : isExplicitAnalyze method may be incorrect in BasicStatsTask (Sergey Shelukhin, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0930154f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0930154f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0930154f Branch: refs/heads/branch-3 Commit: 0930154fdf4ffc35de5bc2fe56cd03a1f48f7598 Parents: 596c811 Author: sergey <ser...@apache.org> Authored: Fri Apr 27 18:57:37 2018 -0700 Committer: sergey <ser...@apache.org> Committed: Fri Apr 27 18:57:50 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java | 1 + .../org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java | 1 + ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java | 1 + .../hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java | 1 + .../java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java | 7 ++++++- 5 files changed, 10 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java index 06b7c93..64f9c70 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java @@ -112,6 +112,7 @@ public class GenMRTableScan1 implements NodeProcessor { // The MR task is just a simple TableScanOperator BasicStatsWork statsWork = new BasicStatsWork(table.getTableSpec()); + statsWork.setIsExplicitAnalyze(true); statsWork.setNoScanAnalyzeCommand(noScan); StatsWork columnStatsWork = new StatsWork(table, statsWork, parseCtx.getConf()); http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java index 5961059..03cceac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java @@ -115,6 +115,7 @@ public class ProcessAnalyzeTable implements NodeProcessor { // The Tez task is just a simple TableScanOperator BasicStatsWork basicStatsWork = new BasicStatsWork(table.getTableSpec()); + basicStatsWork.setIsExplicitAnalyze(true); basicStatsWork.setNoScanAnalyzeCommand(parseContext.getQueryProperties().isNoScanAnalyzeCommand()); StatsWork columnStatsWork = new StatsWork(table, basicStatsWork, parseContext.getConf()); columnStatsWork.collectStatsFromAggregator(tableScan.getConf()); http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index c268ddc..df1d9cb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -392,6 +392,7 @@ public abstract class TaskCompiler { return TaskFactory.get(columnStatsWork); } else { BasicStatsWork statsWork = new BasicStatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); + statsWork.setIsExplicitAnalyze(true); StatsWork columnStatsWork = new StatsWork(table, statsWork, parseContext.getConf()); columnStatsWork.collectStatsFromAggregator(tableScan.getConf()); columnStatsWork.setSourceTask(currentTask); http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java index e81d6f3..28d4de7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java @@ -123,6 +123,7 @@ public class SparkProcessAnalyzeTable implements NodeProcessor { // The Spark task is just a simple TableScanOperator BasicStatsWork basicStatsWork = new BasicStatsWork(table.getTableSpec()); + basicStatsWork.setIsExplicitAnalyze(true); basicStatsWork.setNoScanAnalyzeCommand(parseContext.getQueryProperties().isNoScanAnalyzeCommand()); StatsWork columnStatsWork = new StatsWork(table, basicStatsWork, parseContext.getConf()); columnStatsWork.collectStatsFromAggregator(tableScan.getConf()); http://git-wip-us.apache.org/repos/asf/hive/blob/0930154f/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java index 55d05a1..40def60 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; public class BasicStatsWork implements Serializable { private static final long serialVersionUID = 1L; + private boolean isExplicitAnalyze; private TableSpec tableSpecs; // source table spec -- for TableScanOperator private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator @@ -164,7 +165,7 @@ public class BasicStatsWork implements Serializable { public boolean isExplicitAnalyze() { // ANALYZE TABLE - return (getTableSpecs() != null); + return isExplicitAnalyze; } public boolean isTargetRewritten() { // ANALYZE TABLE @@ -202,4 +203,8 @@ public class BasicStatsWork implements Serializable { } } + public void setIsExplicitAnalyze(boolean b) { + this.isExplicitAnalyze = b; + } + }