HIVE-11467 : WriteBuffers rounding wbSize to next power of 2 may cause OOM (Wei Zheng, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/433ea9cd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/433ea9cd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/433ea9cd Branch: refs/heads/hbase-metastore Commit: 433ea9cda59a986689cf99953826512e4ff07e4d Parents: 1a75644 Author: Sergey Shelukhin <ser...@apache.org> Authored: Thu Aug 13 14:46:07 2015 -0700 Committer: Sergey Shelukhin <ser...@apache.org> Committed: Thu Aug 13 14:46:07 2015 -0700 ---------------------------------------------------------------------- .../hive/ql/exec/persistence/HybridHashTableContainer.java | 4 ++++ serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/433ea9cd/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java index ad1246d..ff64f52 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java @@ -290,6 +290,10 @@ public class HybridHashTableContainer } } + // Round to power of 2 here, as is required by WriteBuffers + writeBufferSize = Integer.bitCount(writeBufferSize) == 1 ? + writeBufferSize : Integer.highestOneBit(writeBufferSize); + // Cap WriteBufferSize to avoid large preallocations writeBufferSize = writeBufferSize < minWbSize ? minWbSize : Math.min(maxWbSize, writeBufferSize); LOG.info("Write buffer size: " + writeBufferSize); http://git-wip-us.apache.org/repos/asf/hive/blob/433ea9cd/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java b/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java index acb51f9..05d9359 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java @@ -56,7 +56,7 @@ public final class WriteBuffers implements RandomAccessOutput { public WriteBuffers(int wbSize, long maxSize) { - this.wbSize = Integer.bitCount(wbSize) == 1 ? wbSize : (Integer.highestOneBit(wbSize) << 1); + this.wbSize = Integer.bitCount(wbSize) == 1 ? wbSize : Integer.highestOneBit(wbSize); this.wbSizeLog2 = 31 - Integer.numberOfLeadingZeros(this.wbSize); this.offsetMask = this.wbSize - 1; this.maxSize = maxSize;