HIVE-17344: LocalCache element memory usage is not calculated properly. (Janos Gub via Zoltan Haindrich, reviewed by Sergey Shelukhin)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/076bd771 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/076bd771 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/076bd771 Branch: refs/heads/hive-14535 Commit: 076bd7716e705132521b267c748ed781b6ec12cb Parents: 8deb779 Author: Janos Gub <gubja...@gmail.com> Authored: Mon Sep 11 09:30:49 2017 +0200 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Mon Sep 11 09:32:52 2017 +0200 ---------------------------------------------------------------------- ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/076bd771/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java index b375aea..e28eb34 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java @@ -48,7 +48,7 @@ class LocalCache implements OrcInputFormat.FooterCache { public long fileLength, fileModTime; public int getMemoryUsage() { - return bb.remaining() + 100; // 100 is for 2 longs, BB and java overheads (semi-arbitrary). + return bb.capacity() + 100; // 100 is for 2 longs, BB and java overheads (semi-arbitrary). } } @@ -78,8 +78,12 @@ class LocalCache implements OrcInputFormat.FooterCache { } public void put(Path path, OrcTail tail) { + ByteBuffer bb = tail.getSerializedTail(); + if (bb.capacity() != bb.remaining()) { + throw new RuntimeException("Bytebuffer allocated for path: " + path + " has remaining: " + bb.remaining() + " != capacity: " + bb.capacity()); + } cache.put(path, new TailAndFileData(tail.getFileTail().getFileLength(), - tail.getFileModificationTime(), tail.getSerializedTail().duplicate())); + tail.getFileModificationTime(), bb.duplicate())); } @Override
