HIVE-18161: Remove hive.stats.atomic (Bertalan Kondrat via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/13c1bf9e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/13c1bf9e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/13c1bf9e Branch: refs/heads/master Commit: 13c1bf9e903c502dbfe3f2fb10959ac1a64bc0ad Parents: 5d4559b Author: Bertalan Kondrat <kb.p...@gmail.com> Authored: Tue Jan 16 11:49:55 2018 +0100 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Tue Jan 16 11:49:55 2018 +0100 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 -- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 3 -- .../hadoop/hive/ql/stats/BasicStatsTask.java | 30 ++------------------ 3 files changed, 3 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 631c836..854bbdf 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1770,8 +1770,6 @@ public class HiveConf extends Configuration { "The Java class (implementing the StatsPublisher interface) that is used by default if hive.stats.dbclass is custom type."), HIVE_STATS_DEFAULT_AGGREGATOR("hive.stats.default.aggregator", "", "The Java class (implementing the StatsAggregator interface) that is used by default if hive.stats.dbclass is custom type."), - HIVE_STATS_ATOMIC("hive.stats.atomic", false, - "whether to update metastore stats only if all stats are available"), CLIENT_STATS_COUNTERS("hive.client.stats.counters", "", "Subset of counters that should be of interest for hive.client.stats.publishers (when one wants to limit their publishing). \n" + "Non-display names should be used"), http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 476c261..a7183ce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -538,9 +538,6 @@ public enum ErrorMsg { STATSAGGREGATOR_SOURCETASK_NULL(30014, "SourceTask of StatsTask should not be null"), STATSAGGREGATOR_CONNECTION_ERROR(30015, "Stats aggregator of type {0} cannot be connected to", true), - STATSAGGREGATOR_MISSED_SOMESTATS(30016, - "Stats type {0} is missing from stats aggregator. If you don't want the query " + - "to fail because of this, set hive.stats.atomic=false", true), STATS_SKIPPING_BY_ERROR(30017, "Skipping stats aggregation by error {0}", true), INVALID_FILE_FORMAT_IN_LOAD(30019, "The file that you are trying to load does not match the" + http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index 21e6984..4e85ae1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -113,13 +113,11 @@ public class BasicStatsTask implements Serializable, IStatsProcessor { private Partish partish; private FileStatus[] partfileStatus; private BasicStatsWork work; - private boolean atomic; private boolean followedColStats1; public BasicStatsProcessor(Partish partish, BasicStatsWork work, HiveConf conf, boolean followedColStats2) { this.partish = partish; this.work = work; - atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC); followedColStats1 = followedColStats2; } @@ -141,12 +139,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor { if (!work.isExplicitAnalyze() && !followedColStats1) { StatsSetupConst.clearColumnStatsState(parameters); } - // non-partitioned tables: - // XXX: I don't aggree with this logic - // FIXME: deprecate atomic? what's its purpose? - if (!existStats(parameters) && atomic) { - return null; - } + if(partfileStatus == null){ LOG.warn("Partition/partfiles is null for: " + partish.getPartition().getSpec()); return null; @@ -164,7 +157,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor { if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { if (statsAggregator != null) { String prefix = getAggregationPrefix(p.getTable(), p.getPartition()); - updateStats(statsAggregator, parameters, prefix, atomic); + updateStats(statsAggregator, parameters, prefix); } } @@ -172,22 +165,9 @@ public class BasicStatsTask implements Serializable, IStatsProcessor { } public void collectFileStatus(Warehouse wh) throws MetaException { - Map<String, String> parameters = partish.getPartParameters(); - if (!existStats(parameters) && atomic) { - return; - } partfileStatus = wh.getFileStatusesForSD(partish.getPartSd()); } - @Deprecated - private boolean existStats(Map<String, String> parameters) { - return parameters.containsKey(StatsSetupConst.ROW_COUNT) - || parameters.containsKey(StatsSetupConst.NUM_FILES) - || parameters.containsKey(StatsSetupConst.TOTAL_SIZE) - || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE) - || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS); - } - private void updateQuickStats(Map<String, String> parameters, FileStatus[] partfileStatus) throws MetaException { MetaStoreUtils.populateQuickStats(partfileStatus, parameters); } @@ -211,7 +191,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor { return prefix; } - private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String aggKey, boolean atomic) throws HiveException { + private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String aggKey) throws HiveException { for (String statType : StatsSetupConst.statsRequireCompute) { String value = statsAggregator.aggregateStats(aggKey, statType); @@ -225,10 +205,6 @@ public class BasicStatsTask implements Serializable, IStatsProcessor { } } parameters.put(statType, String.valueOf(longValue)); - } else { - if (atomic) { - throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); - } } } }