Repository: hive Updated Branches: refs/heads/branch-2 96784f7ca -> 4025eddd4 refs/heads/branch-2.2 9bedccc4b -> 3403535d3 refs/heads/branch-2.3 346f7086c -> 79339c158 refs/heads/master 47c714167 -> 81be9d0d2
HIVE-16547 : LLAP: may not unlock buffers in some cases (Sergey Shelukhin, reviewed by Prasanth Jayachandran) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f4d017b8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f4d017b8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f4d017b8 Branch: refs/heads/master Commit: f4d017b801ea256bf076160a31dba88e61c80422 Parents: 47c7141 Author: sergey <ser...@apache.org> Authored: Thu Apr 27 11:52:07 2017 -0700 Committer: sergey <ser...@apache.org> Committed: Thu Apr 27 11:52:07 2017 -0700 ---------------------------------------------------------------------- .../ql/io/orc/encoded/EncodedReaderImpl.java | 24 ++++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f4d017b8/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java index 326b4b6..5b2e9b5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java @@ -416,8 +416,8 @@ class EncodedReaderImpl implements EncodedReader { if (sctx.stripeLevelStream == null) { sctx.stripeLevelStream = POOLS.csdPool.take(); // We will be using this for each RG while also sending RGs to processing. - // To avoid buffers being unlocked, run refcount one ahead; we will not increase - // it when building the last RG, so each RG processing will decref once, and the + // To avoid buffers being unlocked, run refcount one ahead; so each RG + // processing will decref once, and the // last one will unlock the buffers. sctx.stripeLevelStream.incRef(); // For stripe-level streams we don't need the extra refcount on the block. @@ -430,9 +430,7 @@ class EncodedReaderImpl implements EncodedReader { iter = lastCached; } } - if (!isLastRg) { - sctx.stripeLevelStream.incRef(); - } + sctx.stripeLevelStream.incRef(); cb = sctx.stripeLevelStream; } else { // This stream can be separated by RG using index. Let's do that. @@ -479,6 +477,22 @@ class EncodedReaderImpl implements EncodedReader { } // Release the unreleased buffers. See class comment about refcounts. + for (int colIx = 0; colIx < colCtxs.length; ++colIx) { + ColumnReadContext ctx = colCtxs[colIx]; + if (ctx == null) continue; // This column is not included. + for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) { + StreamContext sctx = ctx.streams[streamIx]; + if (sctx == null || sctx.stripeLevelStream == null) continue; + if (0 != sctx.stripeLevelStream.decRef()) continue; + for (MemoryBuffer buf : sctx.stripeLevelStream.getCacheBuffers()) { + if (LOG.isTraceEnabled()) { + LOG.trace("Unlocking {} at the end of processing", buf); + } + cacheWrapper.releaseBuffer(buf); + } + } + } + releaseInitialRefcounts(toRead.next); // Release buffers as we are done with all the streams... also see toRelease comment. if (toRelease != null) {
