HIVE-10711 Tez HashTableLoader attempts to allocate more memory than available (Mostafa Mokhtar and Jason Dere via Alexander Pivovarov)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1293f3d3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1293f3d3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1293f3d3 Branch: refs/heads/spark Commit: 1293f3d389b73eea672805c49270124f44cbd687 Parents: ac25506 Author: Alexander Pivovarov <apivova...@gmail.com> Authored: Wed May 27 23:30:39 2015 -0700 Committer: Alexander Pivovarov <apivova...@gmail.com> Committed: Wed May 27 23:30:39 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/tez/HashTableLoader.java | 10 ++++++++++ 1 file changed, 10 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1293f3d3/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java index 536b92c..7e67d40 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.tez; import java.io.IOException; +import java.lang.management.ManagementFactory; import java.util.Collections; import java.util.Map; @@ -82,6 +83,15 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable // TODO remove this after memory manager is in long noConditionalTaskThreshold = HiveConf.getLongVar( hconf, HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD); + long processMaxMemory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); + if (noConditionalTaskThreshold > processMaxMemory) { + float hashtableMemoryUsage = HiveConf.getFloatVar( + hconf, HiveConf.ConfVars.HIVEHASHTABLEFOLLOWBYGBYMAXMEMORYUSAGE); + LOG.warn("noConditionalTaskThreshold value of " + noConditionalTaskThreshold + + " is greater than the max memory size of " + processMaxMemory); + // Don't want to attempt to grab more memory than we have available .. percentage is a bit arbitrary + noConditionalTaskThreshold = (long) (processMaxMemory * hashtableMemoryUsage); + } // Only applicable to n-way Hybrid Grace Hash Join HybridHashTableConf nwayConf = null;