http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ParquetMetadataCacheImpl.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ParquetMetadataCacheImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ParquetMetadataCacheImpl.java deleted file mode 100644 index b61a8ca..0000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ParquetMetadataCacheImpl.java +++ /dev/null @@ -1,353 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.llap.io.metadata; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.common.io.FileMetadataCache; -import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; -import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers; -import org.apache.hadoop.hive.llap.cache.EvictionAwareAllocator; -import org.apache.hadoop.hive.llap.cache.EvictionDispatcher; -import org.apache.hadoop.hive.llap.cache.LlapAllocatorBuffer; -import org.apache.hadoop.hive.llap.cache.LlapOomDebugDump; -import org.apache.hadoop.hive.llap.cache.LowLevelCachePolicy; -import org.apache.hadoop.hive.llap.cache.MemoryManager; -import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority; -import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl; -import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; -import org.apache.parquet.io.SeekableInputStream; - -// TODO# merge with OrcMetadataCache (and rename) after HIVE-15665. Shares a lot of the code. -public class ParquetMetadataCacheImpl implements LlapOomDebugDump, FileMetadataCache { - private final ConcurrentHashMap<Object, LlapBufferOrBuffers> metadata = - new ConcurrentHashMap<>(); - - private final MemoryManager memoryManager; - private final LowLevelCachePolicy policy; - private final EvictionAwareAllocator allocator; - private final LlapDaemonCacheMetrics metrics; - - public ParquetMetadataCacheImpl(EvictionAwareAllocator allocator, MemoryManager memoryManager, - LowLevelCachePolicy policy, LlapDaemonCacheMetrics metrics) { - this.memoryManager = memoryManager; - this.allocator = allocator; - this.policy = policy; - this.metrics = metrics; - } - - public void notifyEvicted(LlapFileMetadataBuffer buffer) { - LlapBufferOrBuffers removed = metadata.remove(buffer.getFileKey()); - if (removed == null) return; - if (removed.getSingleBuffer() != null) { - assert removed.getSingleBuffer() == buffer; - return; - } - discardMultiBuffer(removed); - } - - @Override - public LlapBufferOrBuffers getFileMetadata(Object fileKey) { - LlapBufferOrBuffers result = metadata.get(fileKey); - if (result == null) return null; - if (!lockBuffer(result, true)) { - // No need to discard the buffer we cannot lock - eviction takes care of that. - metadata.remove(fileKey, result); - return null; - } - return result; - } - - @Override - public LlapBufferOrBuffers putFileMetadata( - Object fileKey, int length, InputStream is) throws IOException { - LlapBufferOrBuffers result = null; - while (true) { // Overwhelmingly executes once, or maybe twice (replacing stale value). - LlapBufferOrBuffers oldVal = metadata.get(fileKey); - if (oldVal == null) { - result = wrapBbForFile(result, fileKey, length, is); - if (!lockBuffer(result, false)) { - throw new AssertionError("Cannot lock a newly created value " + result); - } - oldVal = metadata.putIfAbsent(fileKey, result); - if (oldVal == null) { - cacheInPolicy(result); // Cached successfully, add to policy. - return result; - } - } - if (lockOldVal(fileKey, result, oldVal)) { - return oldVal; - } - // We found some old value but couldn't incRef it; remove it. - metadata.remove(fileKey, oldVal); - } - } - - private void cacheInPolicy(LlapBufferOrBuffers buffers) { - LlapAllocatorBuffer singleBuffer = buffers.getSingleLlapBuffer(); - if (singleBuffer != null) { - policy.cache(singleBuffer, Priority.HIGH); - return; - } - for (LlapAllocatorBuffer buffer : buffers.getMultipleLlapBuffers()) { - policy.cache(buffer, Priority.HIGH); - } - } - - private <T extends LlapBufferOrBuffers> boolean lockOldVal(Object key, T newVal, T oldVal) { - if (LlapIoImpl.CACHE_LOGGER.isTraceEnabled()) { - LlapIoImpl.CACHE_LOGGER.trace("Trying to cache when metadata is already cached for" + - " {}; old {}, new {}", key, oldVal, newVal); - } - if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) { - LlapIoImpl.LOCKING_LOGGER.trace("Locking {} due to cache collision", oldVal); - } - if (lockBuffer(oldVal, true)) { - // We found an old, valid block for this key in the cache. - if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) { - LlapIoImpl.LOCKING_LOGGER.trace("Unlocking {} due to cache collision with {}", - newVal, oldVal); - } - - if (newVal != null) { - unlockBuffer(newVal, false); - } - return true; - } - return false; - } - - @Override - public void decRefBuffer(MemoryBufferOrBuffers buffer) { - if (!(buffer instanceof LlapBufferOrBuffers)) { - throw new AssertionError(buffer.getClass()); - } - unlockBuffer((LlapBufferOrBuffers)buffer, true); - } - - private LlapBufferOrBuffers wrapBbForFile(LlapBufferOrBuffers result, - Object fileKey, int length, InputStream stream) throws IOException { - if (result != null) return result; - int maxAlloc = allocator.getMaxAllocation(); - LlapFileMetadataBuffer[] largeBuffers = null; - if (maxAlloc < length) { - largeBuffers = new LlapFileMetadataBuffer[length / maxAlloc]; - for (int i = 0; i < largeBuffers.length; ++i) { - largeBuffers[i] = new LlapFileMetadataBuffer(fileKey); - } - allocator.allocateMultiple(largeBuffers, maxAlloc, null); - for (int i = 0; i < largeBuffers.length; ++i) { - readIntoCacheBuffer(stream, maxAlloc, largeBuffers[i]); - } - } - int smallSize = length % maxAlloc; - if (smallSize == 0) { - return new LlapFileMetadataBuffers(largeBuffers); - } else { - LlapFileMetadataBuffer[] smallBuffer = new LlapFileMetadataBuffer[1]; - smallBuffer[0] = new LlapFileMetadataBuffer(fileKey); - allocator.allocateMultiple(smallBuffer, length, null); - readIntoCacheBuffer(stream, smallSize, smallBuffer[0]); - if (largeBuffers == null) { - return smallBuffer[0]; - } else { - LlapFileMetadataBuffer[] cacheData = new LlapFileMetadataBuffer[largeBuffers.length + 1]; - System.arraycopy(largeBuffers, 0, cacheData, 0, largeBuffers.length); - cacheData[largeBuffers.length] = smallBuffer[0]; - return new LlapFileMetadataBuffers(largeBuffers); - } - } - } - - private static void readIntoCacheBuffer( - InputStream stream, int length, MemoryBuffer dest) throws IOException { - ByteBuffer bb = dest.getByteBufferRaw(); - int pos = bb.position(); - bb.limit(pos + length); - // TODO: SeekableInputStream.readFully eventually calls a Hadoop method that used to be - // buggy in 2.7 and also anyway just does a copy for a direct buffer. Do a copy here. - // ((SeekableInputStream)stream).readFully(bb); - FileUtils.readFully(stream, length, bb); - bb.position(pos); - } - - - private boolean lockBuffer(LlapBufferOrBuffers buffers, boolean doNotifyPolicy) { - LlapAllocatorBuffer buffer = buffers.getSingleLlapBuffer(); - if (buffer != null) { - return lockOneBuffer(buffer, doNotifyPolicy); - } - LlapAllocatorBuffer[] bufferArray = buffers.getMultipleLlapBuffers(); - for (int i = 0; i < bufferArray.length; ++i) { - if (lockOneBuffer(bufferArray[i], doNotifyPolicy)) continue; - for (int j = 0; j < i; ++j) { - unlockSingleBuffer(buffer, true); - } - discardMultiBuffer(buffers); - return false; - } - return true; - } - - private void discardMultiBuffer(LlapBufferOrBuffers removed) { - long memoryFreed = 0; - for (LlapAllocatorBuffer buf : removed.getMultipleLlapBuffers()) { - long memUsage = buf.getMemoryUsage(); - // We cannot just deallocate the buffer, as it can hypothetically have users. - int result = buf.invalidate(); - switch (result) { - case LlapAllocatorBuffer.INVALIDATE_ALREADY_INVALID: continue; // Nothing to do. - case LlapAllocatorBuffer.INVALIDATE_FAILED: { - // Someone is using this buffer; eventually, it will be evicted. - continue; - } - case LlapAllocatorBuffer.INVALIDATE_OK: { - memoryFreed += memUsage; - allocator.deallocateEvicted(buf); - break; - } - default: throw new AssertionError(result); - } - } - memoryManager.releaseMemory(memoryFreed); - } - - private boolean lockOneBuffer(LlapAllocatorBuffer buffer, boolean doNotifyPolicy) { - int rc = buffer.incRef(); - if (rc > 0) { - metrics.incrCacheNumLockedBuffers(); - } - if (doNotifyPolicy && rc == 1) { - // We have just locked a buffer that wasn't previously locked. - policy.notifyLock(buffer); - } - return rc > 0; - } - - private void unlockBuffer(LlapBufferOrBuffers buffers, boolean isCached) { - LlapAllocatorBuffer singleBuffer = buffers.getSingleLlapBuffer(); - if (singleBuffer != null) { - unlockSingleBuffer(singleBuffer, isCached); - return; - } - for (LlapAllocatorBuffer buffer : buffers.getMultipleLlapBuffers()) { - unlockSingleBuffer(buffer, isCached); - } - } - - private void unlockSingleBuffer(LlapAllocatorBuffer buffer, boolean isCached) { - boolean isLastDecref = (buffer.decRef() == 0); - if (isLastDecref) { - if (isCached) { - policy.notifyUnlock(buffer); - } else { - allocator.deallocate(buffer); - } - } - metrics.decrCacheNumLockedBuffers(); - } - - - public static interface LlapBufferOrBuffers extends MemoryBufferOrBuffers { - LlapAllocatorBuffer getSingleLlapBuffer(); - LlapAllocatorBuffer[] getMultipleLlapBuffers(); - } - - public final static class LlapFileMetadataBuffer - extends LlapAllocatorBuffer implements LlapBufferOrBuffers { - private final Object fileKey; - - public LlapFileMetadataBuffer(Object fileKey) { - this.fileKey = fileKey; - } - - @Override - public void notifyEvicted(EvictionDispatcher evictionDispatcher) { - evictionDispatcher.notifyEvicted(this); - } - - public Object getFileKey() { - return fileKey; - } - - @Override - public LlapAllocatorBuffer getSingleLlapBuffer() { - return this; - } - - @Override - public LlapAllocatorBuffer[] getMultipleLlapBuffers() { - return null; - } - - @Override - public MemoryBuffer getSingleBuffer() { - return this; - } - - @Override - public MemoryBuffer[] getMultipleBuffers() { - return null; - } - } - - public final static class LlapFileMetadataBuffers implements LlapBufferOrBuffers { - private final LlapFileMetadataBuffer[] buffers; - - public LlapFileMetadataBuffers(LlapFileMetadataBuffer[] buffers) { - this.buffers = buffers; - } - - @Override - public LlapAllocatorBuffer getSingleLlapBuffer() { - return null; - } - - @Override - public LlapAllocatorBuffer[] getMultipleLlapBuffers() { - return buffers; - } - - @Override - public MemoryBuffer getSingleBuffer() { - return null; - } - - @Override - public MemoryBuffer[] getMultipleBuffers() { - return buffers; - } - } - - @Override - public String debugDumpForOom() { - // TODO: nothing, will be merged with ORC cache - return null; - } - - @Override - public void debugDumpShort(StringBuilder sb) { - // TODO: nothing, will be merged with ORC cache - } -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java ---------------------------------------------------------------------- diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java index 13c7767..1ec7020 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java @@ -160,7 +160,7 @@ public class TestIncrementalObjectSizeEstimator { public void close() throws IOException { } } - +/* @Test public void testMetadata() throws IOException { // Mostly tests that it doesn't crash. @@ -202,7 +202,7 @@ public class TestIncrementalObjectSizeEstimator { root = map.get(OrcFileMetadata.class); LOG.info("Estimated " + root.estimate(ofm, map) + " for a dummy OFM"); - } + }*/ private static class Struct { Integer i; http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java ---------------------------------------------------------------------- diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java index 03a955c..3ebb4b0 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java @@ -19,10 +19,18 @@ package org.apache.hadoop.hive.llap.cache; import static org.junit.Assert.*; +import java.nio.ByteBuffer; +import java.util.Random; + +import org.apache.hadoop.hive.common.io.DataCache; +import org.apache.hadoop.hive.common.io.DiskRange; +import org.apache.hadoop.hive.common.io.DiskRangeList; import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority; -import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata; -import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache; -import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata; +import org.apache.hadoop.hive.llap.io.metadata.MetadataCache; +import org.apache.hadoop.hive.llap.io.metadata.MetadataCache.LlapBufferOrBuffers; +import org.apache.hadoop.hive.llap.io.metadata.MetadataCache.LlapMetadataBuffer; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; +import org.apache.hadoop.hive.ql.io.orc.encoded.IncompleteCb; import org.junit.Test; public class TestOrcMetadataCache { @@ -66,16 +74,12 @@ public class TestOrcMetadataCache { } private static class DummyMemoryManager implements MemoryManager { - int allocs = 0; - @Override public void reserveMemory(long memoryToReserve) { - ++allocs; } @Override public void releaseMemory(long memUsage) { - --allocs; } @Override @@ -93,38 +97,119 @@ public class TestOrcMetadataCache { } @Test - public void testGetPut() throws Exception { + public void testBuffers() throws Exception { + DummyMemoryManager mm = new DummyMemoryManager(); + DummyCachePolicy cp = new DummyCachePolicy(); + final int MAX_ALLOC = 64; + LlapDaemonCacheMetrics metrics = LlapDaemonCacheMetrics.create("", ""); + BuddyAllocator alloc = new BuddyAllocator( + false, false, 8, MAX_ALLOC, 1, 4096, 0, null, mm, metrics, null); + MetadataCache cache = new MetadataCache(alloc, mm, cp, true, metrics); + Object fileKey1 = new Object(); + Random rdm = new Random(); + + ByteBuffer smallBuffer = ByteBuffer.allocate(MAX_ALLOC - 1); + rdm.nextBytes(smallBuffer.array()); + LlapBufferOrBuffers result = cache.putFileMetadata(fileKey1, smallBuffer); + cache.decRefBuffer(result); + ByteBuffer cacheBuf = result.getSingleBuffer().getByteBufferDup(); + assertEquals(smallBuffer, cacheBuf); + result = cache.putFileMetadata(fileKey1, smallBuffer); + cache.decRefBuffer(result); + cacheBuf = result.getSingleBuffer().getByteBufferDup(); + assertEquals(smallBuffer, cacheBuf); + result = cache.getFileMetadata(fileKey1); + cacheBuf = result.getSingleBuffer().getByteBufferDup(); + assertEquals(smallBuffer, cacheBuf); + cache.decRefBuffer(result); + cache.notifyEvicted((LlapMetadataBuffer<?>) result.getSingleBuffer()); + result = cache.getFileMetadata(fileKey1); + assertNull(result); + + ByteBuffer largeBuffer = ByteBuffer.allocate((int)(MAX_ALLOC * 2.5)); + rdm.nextBytes(largeBuffer.array()); + result = cache.putFileMetadata(fileKey1, largeBuffer); + cache.decRefBuffer(result); + assertNull(result.getSingleBuffer()); + assertEquals(largeBuffer, extractResultBbs(result)); + result = cache.getFileMetadata(fileKey1); + assertNull(result.getSingleBuffer()); + assertEquals(largeBuffer, extractResultBbs(result)); + LlapAllocatorBuffer b0 = result.getMultipleLlapBuffers()[0], + b1 = result.getMultipleLlapBuffers()[1]; + cache.decRefBuffer(result); + cache.notifyEvicted((LlapMetadataBuffer<?>) b1); + result = cache.getFileMetadata(fileKey1); + assertNull(result); + assertFalse(b0.incRef() > 0); // Should have also been thrown out. + } + + public ByteBuffer extractResultBbs(LlapBufferOrBuffers result) { + int totalLen = 0; + for (LlapAllocatorBuffer buf : result.getMultipleLlapBuffers()) { + totalLen += buf.getByteBufferRaw().remaining(); + } + ByteBuffer combinedBb = ByteBuffer.allocate(totalLen); + for (LlapAllocatorBuffer buf : result.getMultipleLlapBuffers()) { + combinedBb.put(buf.getByteBufferDup()); + } + combinedBb.flip(); + return combinedBb; + } + + @Test + public void testIncompleteCbs() throws Exception { DummyMemoryManager mm = new DummyMemoryManager(); DummyCachePolicy cp = new DummyCachePolicy(); - OrcMetadataCache cache = new OrcMetadataCache(mm, cp, false); - OrcFileMetadata ofm1 = OrcFileMetadata.createDummy(1), ofm2 = OrcFileMetadata.createDummy(2); - assertSame(ofm1, cache.putFileMetadata(ofm1)); - assertEquals(1, mm.allocs); + final int MAX_ALLOC = 64; + LlapDaemonCacheMetrics metrics = LlapDaemonCacheMetrics.create("", ""); + BuddyAllocator alloc = new BuddyAllocator( + false, false, 8, MAX_ALLOC, 1, 4096, 0, null, mm, metrics, null); + MetadataCache cache = new MetadataCache(alloc, mm, cp, true, metrics); + DataCache.BooleanRef gotAllData = new DataCache.BooleanRef(); + Object fileKey1 = new Object(); + + // Note: incomplete CBs are always an exact match. + cache.putIncompleteCbs(fileKey1, new DiskRange[] { new DiskRangeList(0, 3) }, 0); cp.verifyEquals(1); - assertSame(ofm2, cache.putFileMetadata(ofm2)); - assertEquals(2, mm.allocs); - cp.verifyEquals(2); - assertSame(ofm1, cache.getFileMetadata(1)); - assertSame(ofm2, cache.getFileMetadata(2)); - cp.verifyEquals(4); - OrcFileMetadata ofm3 = OrcFileMetadata.createDummy(1); - assertSame(ofm1, cache.putFileMetadata(ofm3)); - assertEquals(2, mm.allocs); - cp.verifyEquals(5); - assertSame(ofm1, cache.getFileMetadata(1)); - cp.verifyEquals(6); - - OrcStripeMetadata osm1 = OrcStripeMetadata.createDummy(1), osm2 = OrcStripeMetadata.createDummy(2); - assertSame(osm1, cache.putStripeMetadata(osm1)); - assertEquals(3, mm.allocs); - assertSame(osm2, cache.putStripeMetadata(osm2)); - assertEquals(4, mm.allocs); - assertSame(osm1, cache.getStripeMetadata(osm1.getKey())); - assertSame(osm2, cache.getStripeMetadata(osm2.getKey())); - OrcStripeMetadata osm3 = OrcStripeMetadata.createDummy(1); - assertSame(osm1, cache.putStripeMetadata(osm3)); - assertEquals(4, mm.allocs); - assertSame(osm1, cache.getStripeMetadata(osm3.getKey())); - cp.verifyEquals(12); + DiskRangeList result = cache.getIncompleteCbs( + fileKey1, new DiskRangeList(0, 3), 0, gotAllData); + assertTrue(gotAllData.value); + verifyResult(result, INCOMPLETE, 0, 3); + cache.putIncompleteCbs(fileKey1, new DiskRange[] { new DiskRangeList(5, 6) }, 0); + cp.verifyEquals(3); + DiskRangeList ranges = new DiskRangeList(0, 3); + ranges.insertAfter(new DiskRangeList(4, 6)); + result = cache.getIncompleteCbs(fileKey1, ranges, 0, gotAllData); + assertFalse(gotAllData.value); + verifyResult(result, INCOMPLETE, 0, 3, DRL, 4, 6); + ranges = new DiskRangeList(0, 3); + ranges.insertAfter(new DiskRangeList(3, 5)).insertAfter(new DiskRangeList(5, 6)); + result = cache.getIncompleteCbs(fileKey1, ranges, 0, gotAllData); + assertFalse(gotAllData.value); + verifyResult(result, INCOMPLETE, 0, 3, DRL, 3, 5, INCOMPLETE, 5, 6); + result = cache.getIncompleteCbs(fileKey1, new DiskRangeList(5, 6), 0, gotAllData); + assertTrue(gotAllData.value); + verifyResult(result, INCOMPLETE, 5, 6); + result = cache.getIncompleteCbs(fileKey1, new DiskRangeList(4, 5), 0, gotAllData); + assertFalse(gotAllData.value); + verifyResult(result, DRL, 4, 5); + } + + private static final int INCOMPLETE = 0, DRL = 1; + public void verifyResult(DiskRangeList result, long... vals) { + for (int i = 0; i < vals.length; i += 3) { + switch ((int)vals[i]) { + case INCOMPLETE: assertTrue(result instanceof IncompleteCb); break; + case DRL: assertFalse(result instanceof IncompleteCb); break; + default: fail(); + } + assertEquals(vals[i + 1], result.getOffset()); + assertEquals(vals[i + 2], result.getEnd()); + result = result.next; + } + assertNull(result); } + + } http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 69a9f9f..c5d9b7b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -112,6 +112,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hive.common.util.Ref; import org.apache.orc.ColumnStatistics; import org.apache.orc.OrcProto; +import org.apache.orc.OrcProto.Footer; import org.apache.orc.OrcUtils; import org.apache.orc.StripeInformation; import org.apache.orc.StripeStatistics; @@ -330,6 +331,16 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, return !file.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME); } + public static boolean isOriginal(Footer footer) { + for(OrcProto.UserMetadataItem item: footer.getMetadataList()) { + if (item.hasName() && item.getName().equals(OrcRecordUpdater.ACID_KEY_INDEX_NAME)) { + return true; + } + } + return false; + } + + public static boolean[] genIncludedColumns(TypeDescription readerSchema, List<Integer> included) { http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java index 7540e72..dcfb577 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.orc.StripeInformation; import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch; import org.apache.orc.OrcProto; +import org.apache.orc.impl.OrcIndex; public interface EncodedReader { @@ -54,4 +55,17 @@ public interface EncodedReader { * to just checking the constant in the first place. */ void setTracing(boolean isEnabled); + + /** + * Read the indexes from ORC file. + * @param index The destination with pre-allocated arrays to put index data into. + * @param stripe Externally provided metadata (from metadata reader or external cache). + * @param streams Externally provided metadata (from metadata reader or external cache). + * @param included The array of booleans indicating whether each column should be read. + * @param sargColumns The array of booleans indicating whether each column's + * bloom filters should be read. + */ + void readIndexStreams(OrcIndex index, StripeInformation stripe, + List<OrcProto.Stream> streams, boolean[] included, boolean[] sargColumns) + throws IOException; } http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java index 690cce7..467bac2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.io.orc.encoded; import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -35,16 +36,23 @@ import org.apache.hadoop.hive.common.io.DiskRangeList; import org.apache.hadoop.hive.common.io.DataCache.BooleanRef; import org.apache.hadoop.hive.common.io.DataCache.DiskRangeListFactory; import org.apache.hadoop.hive.common.io.DiskRangeList.CreateHelper; +import org.apache.hadoop.hive.common.io.DiskRangeList.MutateHelper; import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData; import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; import org.apache.orc.CompressionCodec; import org.apache.orc.DataReader; import org.apache.orc.OrcConf; +import org.apache.orc.OrcFile.WriterVersion; import org.apache.orc.OrcProto.ColumnEncoding; +import org.apache.orc.OrcProto.Stream; import org.apache.orc.OrcProto.Stream.Kind; +import org.apache.orc.TypeDescription; +import org.apache.orc.impl.InStream; +import org.apache.orc.impl.OrcIndex; import org.apache.orc.impl.OutStream; import org.apache.orc.impl.RecordReaderUtils; import org.apache.orc.impl.StreamName; +import org.apache.orc.impl.StreamName.Area; import org.apache.orc.StripeInformation; import org.apache.orc.impl.BufferChunk; import org.apache.hadoop.hive.ql.io.orc.encoded.IoTrace.RangesSrc; @@ -53,6 +61,7 @@ import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.PoolFactory; import org.apache.orc.OrcProto; import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.CodedInputStream; import sun.misc.Cleaner; @@ -126,13 +135,18 @@ class EncodedReaderImpl implements EncodedReader { private final DataCache cacheWrapper; private boolean isTracingEnabled; private final IoTrace trace; + private final TypeDescription fileSchema; + private final WriterVersion version; - public EncodedReaderImpl(Object fileKey, List<OrcProto.Type> types, CompressionCodec codec, + public EncodedReaderImpl(Object fileKey, List<OrcProto.Type> types, + TypeDescription fileSchema, CompressionCodec codec, WriterVersion version, int bufferSize, long strideRate, DataCache cacheWrapper, DataReader dataReader, PoolFactory pf, IoTrace trace) throws IOException { this.fileKey = fileKey; this.codec = codec; this.types = types; + this.fileSchema = fileSchema; // Note: this is redundant with types + this.version = version; this.bufferSize = bufferSize; this.rowIndexStride = strideRate; this.cacheWrapper = cacheWrapper; @@ -150,25 +164,60 @@ class EncodedReaderImpl implements EncodedReader { } /** Helper context for each column being read */ - private static final class ColumnReadContext { + private static final class ColumnReadContext extends ReadContext { public ColumnReadContext(int colIx, OrcProto.ColumnEncoding encoding, OrcProto.RowIndex rowIndex, int colRgIx) { + super(colIx, colRgIx, MAX_STREAMS); this.encoding = encoding; this.rowIndex = rowIndex; + } + + public static final int MAX_STREAMS = countMaxStreams(Area.DATA); + /** Column encoding. */ + OrcProto.ColumnEncoding encoding; + /** Column rowindex. */ + OrcProto.RowIndex rowIndex; + + public void addStream(long offset, OrcProto.Stream stream, int indexIx) { + streams[streamCount++] = new StreamContext(stream, offset, indexIx); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(" column_index: ").append(colIx); + sb.append(" included_index: ").append(includedIx); + sb.append(" encoding: ").append(encoding); + sb.append(" stream_count: ").append(streamCount); + int i = 0; + for (StreamContext sc : streams) { + if (sc != null) { + sb.append(" stream_").append(i).append(":").append(sc.toString()); + } + i++; + } + return sb.toString(); + } + } + + /** Helper context for each column for which the index is being read */ + private static class ReadContext { + protected ReadContext(int colIx, int colRgIx, int maxStreams) { this.colIx = colIx; this.includedIx = colRgIx; streamCount = 0; + streams = new StreamContext[maxStreams]; } - public static final int MAX_STREAMS = OrcProto.Stream.Kind.ROW_INDEX_VALUE; + public ReadContext(int colIx, int colRgIx) { + this(colIx, colRgIx, MAX_STREAMS); + } + + public static final int MAX_STREAMS = countMaxStreams(Area.INDEX); /** The number of streams that are part of this column. */ int streamCount = 0; - final StreamContext[] streams = new StreamContext[MAX_STREAMS]; - /** Column encoding. */ - OrcProto.ColumnEncoding encoding; - /** Column rowindex. */ - OrcProto.RowIndex rowIndex; + final StreamContext[] streams; /** Column index in the file. */ int colIx; /** Column index in the included columns only (for RG masks). */ @@ -183,7 +232,6 @@ class EncodedReaderImpl implements EncodedReader { StringBuilder sb = new StringBuilder(); sb.append(" column_index: ").append(colIx); sb.append(" included_index: ").append(includedIx); - sb.append(" encoding: ").append(encoding); sb.append(" stream_count: ").append(streamCount); int i = 0; for (StreamContext sc : streams) { @@ -313,56 +361,10 @@ class EncodedReaderImpl implements EncodedReader { } // 2. Now, read all of the ranges from cache or disk. - DiskRangeList.MutateHelper toRead = new DiskRangeList.MutateHelper(listToRead.get()); - if (LOG.isDebugEnabled()) { - LOG.debug("Resulting disk ranges to read (file " + fileKey + "): " - + RecordReaderUtils.stringifyDiskRanges(toRead.next)); - } - trace.logRanges(fileKey, stripeOffset, toRead.next, RangesSrc.PLAN); - BooleanRef isAllInCache = new BooleanRef(); - if (hasFileId) { - cacheWrapper.getFileData(fileKey, toRead.next, stripeOffset, CC_FACTORY, isAllInCache); - if (LOG.isDebugEnabled()) { - LOG.debug("Disk ranges after cache (found everything " + isAllInCache.value + "; file " - + fileKey + ", base offset " + stripeOffset + "): " - + RecordReaderUtils.stringifyDiskRanges(toRead.next)); - } - trace.logRanges(fileKey, stripeOffset, toRead.next, RangesSrc.CACHE); - } + IdentityHashMap<ByteBuffer, Boolean> toRelease = new IdentityHashMap<>(); + MutateHelper toRead = getDataFromCacheAndDisk( + listToRead.get(), stripeOffset, hasFileId, toRelease); - // TODO: the memory release could be optimized - we could release original buffers after we - // are fully done with each original buffer from disk. For now release all at the end; - // it doesn't increase the total amount of memory we hold, just the duration a bit. - // This is much simpler - we can just remember original ranges after reading them, and - // release them at the end. In a few cases where it's easy to determine that a buffer - // can be freed in advance, we remove it from the map. - IdentityHashMap<ByteBuffer, Boolean> toRelease = null; - if (!isAllInCache.value) { - boolean hasError = true; - try { - if (!isDataReaderOpen) { - this.dataReader.open(); - isDataReaderOpen = true; - } - dataReader.readFileData(toRead.next, stripeOffset, - cacheWrapper.getAllocator().isDirectAlloc()); - toRelease = new IdentityHashMap<>(); - DiskRangeList drl = toRead.next; - while (drl != null) { - if (drl instanceof BufferChunk) { - toRelease.put(drl.getData(), true); - } - drl = drl.next; - } - hasError = false; - } finally { - // The FS can be closed from under us if the task is interrupted. Release cache buffers. - // We are assuming here that toRelease will not be present in such cases. - if (hasError) { - releaseInitialRefcounts(toRead.next); - } - } - } // 3. For uncompressed case, we need some special processing before read. // Basically, we are trying to create artificial, consistent ranges to cache, as there are @@ -370,46 +372,7 @@ class EncodedReaderImpl implements EncodedReader { // either cache buffers, or buffers allocated by us and not cached (if we are only reading // parts of the data for some ranges and don't want to cache it). Both are represented by // CacheChunks, so the list is just CacheChunk-s from that point on. - DiskRangeList iter = toRead.next; - if (codec == null) { - boolean hasError = true; - try { - for (int colIx = 0; colIx < colCtxs.length; ++colIx) { - ColumnReadContext ctx = colCtxs[colIx]; - if (ctx == null) continue; // This column is not included. - for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) { - StreamContext sctx = ctx.streams[streamIx]; - DiskRangeList newIter = preReadUncompressedStream( - stripeOffset, iter, sctx.offset, sctx.offset + sctx.length, sctx.kind); - if (newIter != null) { - iter = newIter; - } - } - } - // Release buffers as we are done with all the streams... also see toRelease comment. - // With uncompressed streams, we know we are done earlier. - if (toRelease != null) { - releaseBuffers(toRelease.keySet(), true); - toRelease = null; - } - if (LOG.isInfoEnabled()) { - LOG.info("Disk ranges after pre-read (file " + fileKey + ", base offset " - + stripeOffset + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next)); - } - iter = toRead.next; // Reset the iter to start. - hasError = false; - } finally { - // At this point, everything in the list is going to have a refcount of one. Unless it - // failed between the allocation and the incref for a single item, we should be ok. - if (hasError) { - releaseInitialRefcounts(toRead.next); - if (toRelease != null) { - releaseBuffers(toRelease.keySet(), true); - toRelease = null; - } - } - } - } + DiskRangeList iter = preReadUncompressedStreams(stripeOffset, colCtxs, toRead, toRelease); try { // 4. Finally, decompress data, map per RG, and return to caller. @@ -542,13 +505,74 @@ class EncodedReaderImpl implements EncodedReader { releaseInitialRefcounts(toRead.next); // Release buffers as we are done with all the streams... also see toRelease comment. - if (toRelease != null) { - releaseBuffers(toRelease.keySet(), true); - } + releaseBuffers(toRelease.keySet(), true); } releaseCacheChunksIntoObjectPool(toRead.next); } + private static int countMaxStreams(Area area) { + int count = 0; + for (Stream.Kind sk : Stream.Kind.values()) { + if (StreamName.getArea(sk) == area) { + ++count; + } + } + return count; + } + + private DiskRangeList.MutateHelper getDataFromCacheAndDisk(DiskRangeList listToRead, + long stripeOffset, boolean hasFileId, IdentityHashMap<ByteBuffer, Boolean> toRelease) + throws IOException { + DiskRangeList.MutateHelper toRead = new DiskRangeList.MutateHelper(listToRead); + if (LOG.isInfoEnabled()) { + LOG.info("Resulting disk ranges to read (file " + fileKey + "): " + + RecordReaderUtils.stringifyDiskRanges(toRead.next)); + } + BooleanRef isAllInCache = new BooleanRef(); + if (hasFileId) { + cacheWrapper.getFileData(fileKey, toRead.next, stripeOffset, CC_FACTORY, isAllInCache); + if (LOG.isInfoEnabled()) { + LOG.info("Disk ranges after cache (found everything " + isAllInCache.value + "; file " + + fileKey + ", base offset " + stripeOffset + "): " + + RecordReaderUtils.stringifyDiskRanges(toRead.next)); + } + trace.logRanges(fileKey, stripeOffset, toRead.next, RangesSrc.CACHE); + } + + // TODO: the memory release could be optimized - we could release original buffers after we + // are fully done with each original buffer from disk. For now release all at the end; + // it doesn't increase the total amount of memory we hold, just the duration a bit. + // This is much simpler - we can just remember original ranges after reading them, and + // release them at the end. In a few cases where it's easy to determine that a buffer + // can be freed in advance, we remove it from the map. + if (!isAllInCache.value) { + boolean hasError = true; + try { + if (!isDataReaderOpen) { + this.dataReader.open(); + isDataReaderOpen = true; + } + dataReader.readFileData(toRead.next, stripeOffset, + cacheWrapper.getAllocator().isDirectAlloc()); + toRelease = new IdentityHashMap<>(); + DiskRangeList drl = toRead.next; + while (drl != null) { + if (drl instanceof BufferChunk) { + toRelease.put(drl.getData(), true); + } + drl = drl.next; + } + hasError = false; + } finally { + // The FS can be closed from under us if the task is interrupted. Release cache buffers. + // We are assuming here that toRelease will not be present in such cases. + if (hasError) { + releaseInitialRefcounts(toRead.next); + } + } + } + return toRead; + } private void releaseEcbRefCountsOnError(OrcEncodedColumnBatch ecb) { if (isTracingEnabled) { @@ -805,7 +829,8 @@ class EncodedReaderImpl implements EncodedReader { targetBuffers[ix] = chunk.getBuffer(); ++ix; } - cacheWrapper.getAllocator().allocateMultiple(targetBuffers, bufferSize); + cacheWrapper.getAllocator().allocateMultiple(targetBuffers, bufferSize, + cacheWrapper.getDataBufferFactory()); // 4. Now decompress (or copy) the data into cache buffers. for (ProcCacheChunk chunk : toDecompress) { @@ -1067,8 +1092,8 @@ class EncodedReaderImpl implements EncodedReader { cacheKeys[ix] = chunk; // Relies on the fact that cache does not actually store these. ++ix; } - cacheWrapper.getAllocator().allocateMultiple( - targetBuffers, (int)(partCount == 1 ? streamLen : partSize)); + cacheWrapper.getAllocator().allocateMultiple(targetBuffers, + (int)(partCount == 1 ? streamLen : partSize), cacheWrapper.getDataBufferFactory()); // 4. Now copy the data into cache buffers. ix = 0; @@ -1120,7 +1145,8 @@ class EncodedReaderImpl implements EncodedReader { // non-cached. Since we are at the first gap, the previous stuff must be contiguous. singleAlloc[0] = null; trace.logPartialUncompressedData(partOffset, candidateEnd, true); - cacheWrapper.getAllocator().allocateMultiple(singleAlloc, (int)(candidateEnd - partOffset)); + cacheWrapper.getAllocator().allocateMultiple( + singleAlloc, (int)(candidateEnd - partOffset), cacheWrapper.getDataBufferFactory()); MemoryBuffer buffer = singleAlloc[0]; cacheWrapper.reuseBuffer(buffer); ByteBuffer dest = buffer.getByteBufferRaw(); @@ -1134,7 +1160,8 @@ class EncodedReaderImpl implements EncodedReader { BufferChunk bc, DataCache cacheWrapper, MemoryBuffer[] singleAlloc) { singleAlloc[0] = null; trace.logPartialUncompressedData(bc.getOffset(), bc.getEnd(), false); - cacheWrapper.getAllocator().allocateMultiple(singleAlloc, bc.getLength()); + cacheWrapper.getAllocator().allocateMultiple( + singleAlloc, bc.getLength(), cacheWrapper.getDataBufferFactory()); MemoryBuffer buffer = singleAlloc[0]; cacheWrapper.reuseBuffer(buffer); ByteBuffer dest = buffer.getByteBufferRaw(); @@ -1597,7 +1624,7 @@ class EncodedReaderImpl implements EncodedReader { BufferChunk lastChunk, List<ProcCacheChunk> toDecompress, List<MemoryBuffer> cacheBuffers, boolean doTrace) { // Prepare future cache buffer. - MemoryBuffer futureAlloc = cacheWrapper.getAllocator().createUnallocated(); + MemoryBuffer futureAlloc = cacheWrapper.getDataBufferFactory().create(); // Add it to result in order we are processing. cacheBuffers.add(futureAlloc); // Add it to the list of work to decompress. @@ -1706,4 +1733,297 @@ class EncodedReaderImpl implements EncodedReader { }); } } + + // TODO: perhaps move to Orc InStream? + private static class IndexStream extends InputStream { + private List<MemoryBuffer> ranges; + private long currentOffset = 0, length; + private ByteBuffer range; + private int rangeIx = -1; + + public IndexStream(List<MemoryBuffer> input, long length) { + this.ranges = input; + this.length = length; + } + + @Override + public int read() { + if (!ensureRangeWithData()) { + return -1; + } + currentOffset += 1; + return 0xff & range.get(); + } + + private boolean ensureRangeWithData() { + if (range != null && range.remaining() > 0) return true; + ++rangeIx; + if (rangeIx == ranges.size()) return false; + range = ranges.get(rangeIx).getByteBufferDup(); + return true; + } + + @Override + public int read(byte[] data, int offset, int length) { + if (!ensureRangeWithData()) { + return -1; + } + int actualLength = Math.min(length, range.remaining()); + range.get(data, offset, actualLength); + currentOffset += actualLength; + return actualLength; + } + + @Override + public int available() { + if (range != null && range.remaining() > 0) { + return range.remaining(); + } + return (int) (length - currentOffset); + } + + @Override + public void close() { + rangeIx = ranges.size(); + currentOffset = length; + ranges.clear(); + } + + @Override + public String toString() { + return "position: " + currentOffset + " length: " + length + " range: " + rangeIx + + " offset: " + (range == null ? 0 : range.position()) + + " limit: " + (range == null ? 0 : range.limit()); + } + } + + @Override + public void readIndexStreams(OrcIndex index, StripeInformation stripe, + List<OrcProto.Stream> streams, boolean[] included, boolean[] sargColumns) + throws IOException { + long stripeOffset = stripe.getOffset(); + DiskRangeList indexRanges = planIndexReading( + fileSchema, streams, true, included, sargColumns, version, index.getBloomFilterKinds()); + if (indexRanges == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Nothing to read for stripe [" + stripe + "]"); + } + return; + } + ReadContext[] colCtxs = new ReadContext[included.length]; + int colRgIx = -1; + for (int i = 0; i < included.length; ++i) { + if (!included[i] && (sargColumns == null || !sargColumns[i])) continue; + colCtxs[i] = new ReadContext(i, ++colRgIx); + if (isTracingEnabled) { + LOG.trace("Creating context: " + colCtxs[i].toString()); + } + } + long offset = 0; + for (OrcProto.Stream stream : streams) { + long length = stream.getLength(); + int colIx = stream.getColumn(); + OrcProto.Stream.Kind streamKind = stream.getKind(); + // See planIndexReading - only read non-row-index streams if involved in SARGs. + if ((StreamName.getArea(streamKind) == StreamName.Area.INDEX) + && ((sargColumns != null && sargColumns[colIx]) + || (included[colIx] && streamKind == Kind.ROW_INDEX))) { + colCtxs[colIx].addStream(offset, stream, -1); + if (isTracingEnabled) { + LOG.trace("Adding stream for column " + colIx + ": " + + streamKind + " at " + offset + ", " + length); + } + } + offset += length; + } + + boolean hasFileId = this.fileKey != null; + + // 2. Now, read all of the ranges from cache or disk. + IdentityHashMap<ByteBuffer, Boolean> toRelease = new IdentityHashMap<>(); + MutateHelper toRead = getDataFromCacheAndDisk(indexRanges, stripeOffset, hasFileId, toRelease); + + // 3. For uncompressed case, we need some special processing before read. + DiskRangeList iter = preReadUncompressedStreams(stripeOffset, colCtxs, toRead, toRelease); + + // 4. Decompress the data. + for (int colIx = 0; colIx < colCtxs.length; ++colIx) { + ReadContext ctx = colCtxs[colIx]; + if (ctx == null) continue; // This column is not included. + for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) { + StreamContext sctx = ctx.streams[streamIx]; + try { + if (isTracingEnabled) { + LOG.trace("Getting index stream " + sctx.kind + " for column " + ctx.colIx + + " at " + sctx.offset + ", " + sctx.length); + } + ColumnStreamData csd = POOLS.csdPool.take(); + long endCOffset = sctx.offset + sctx.length; + DiskRangeList lastCached = readEncodedStream(stripeOffset, iter, sctx.offset, + endCOffset, csd, endCOffset, sctx.offset, toRelease); + if (lastCached != null) { + iter = lastCached; + } + CodedInputStream cis = CodedInputStream.newInstance( + new IndexStream(csd.getCacheBuffers(), sctx.length)); + cis.setSizeLimit(InStream.PROTOBUF_MESSAGE_MAX_LIMIT); + switch (sctx.kind) { + case ROW_INDEX: + index.getRowGroupIndex()[colIx] = OrcProto.RowIndex.parseFrom(cis); + break; + case BLOOM_FILTER: + case BLOOM_FILTER_UTF8: + index.getBloomFilterIndex()[colIx] = OrcProto.BloomFilterIndex.parseFrom(cis); + break; + default: + throw new AssertionError("Unexpected index stream type " + sctx.kind); + } + // We are done with the buffers; unlike data blocks, we are also the consumer. Release. + for (MemoryBuffer buf : csd.getCacheBuffers()) { + if (buf == null) continue; + cacheWrapper.releaseBuffer(buf); + } + } catch (Exception ex) { + DiskRangeList drl = toRead == null ? null : toRead.next; + LOG.error("Error getting stream " + sctx.kind + " for column " + ctx.colIx + + " at " + sctx.offset + ", " + sctx.length + "; toRead " + + RecordReaderUtils.stringifyDiskRanges(drl), ex); + throw (ex instanceof IOException) ? (IOException)ex : new IOException(ex); + } + } + } + + if (isTracingEnabled) { + LOG.trace("Disk ranges after preparing all the data " + + RecordReaderUtils.stringifyDiskRanges(toRead.next)); + } + + // Release the unreleased buffers. See class comment about refcounts. + releaseInitialRefcounts(toRead.next); + releaseBuffers(toRelease.keySet(), true); + releaseCacheChunksIntoObjectPool(toRead.next); + } + + + private DiskRangeList preReadUncompressedStreams(long stripeOffset, ReadContext[] colCtxs, + MutateHelper toRead, IdentityHashMap<ByteBuffer, Boolean> toRelease) throws IOException { + if (codec != null) return toRead.next; + DiskRangeList iter = toRead.next; // Keep "toRead" list for future use, don't extract(). + boolean hasError = true; + try { + for (int colIx = 0; colIx < colCtxs.length; ++colIx) { + ReadContext ctx = colCtxs[colIx]; + if (ctx == null) continue; // This column is not included. + for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) { + StreamContext sctx = ctx.streams[streamIx]; + DiskRangeList newIter = preReadUncompressedStream( + stripeOffset, iter, sctx.offset, sctx.offset + sctx.length, sctx.kind); + if (newIter != null) { + iter = newIter; + } + } + } + // Release buffers as we are done with all the streams... also see toRelease comment.\ + // With uncompressed streams, we know we are done earlier. + if (toRelease != null) { + releaseBuffers(toRelease.keySet(), true); + toRelease.clear(); + } + if (LOG.isInfoEnabled()) { + LOG.info("Disk ranges after pre-read (file " + fileKey + ", base offset " + + stripeOffset + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next)); + } + iter = toRead.next; // Reset the iter to start. + hasError = false; + } finally { + // At this point, everything in the list is going to have a refcount of one. Unless it + // failed between the allocation and the incref for a single item, we should be ok. + if (hasError) { + releaseInitialRefcounts(toRead.next); + if (toRelease != null) { + releaseBuffers(toRelease.keySet(), true); + toRelease.clear(); + } + } + } + return toRead.next; // Reset the iter to start. + } + + // TODO: temporary, need to expose from ORC utils (note the difference in null checks) + static DiskRangeList planIndexReading(TypeDescription fileSchema, + List<OrcProto.Stream> streams, + boolean ignoreNonUtf8BloomFilter, + boolean[] fileIncluded, + boolean[] sargColumns, + WriterVersion version, + OrcProto.Stream.Kind[] bloomFilterKinds) { + DiskRangeList.CreateHelper result = new DiskRangeList.CreateHelper(); + // figure out which kind of bloom filter we want for each column + // picks bloom_filter_utf8 if its available, otherwise bloom_filter + if (sargColumns != null) { + for (OrcProto.Stream stream : streams) { + if (stream.hasKind() && stream.hasColumn()) { + int column = stream.getColumn(); + if (sargColumns[column]) { + switch (stream.getKind()) { + case BLOOM_FILTER: + if (bloomFilterKinds[column] == null && + !(ignoreNonUtf8BloomFilter && + hadBadBloomFilters(fileSchema.findSubtype(column) + .getCategory(), version))) { + bloomFilterKinds[column] = OrcProto.Stream.Kind.BLOOM_FILTER; + } + break; + case BLOOM_FILTER_UTF8: + bloomFilterKinds[column] = OrcProto.Stream.Kind.BLOOM_FILTER_UTF8; + break; + default: + break; + } + } + } + } + } + long offset = 0; + for (OrcProto.Stream stream: streams) { + if (stream.hasKind() && stream.hasColumn()) { + int column = stream.getColumn(); + if (fileIncluded == null || fileIncluded[column]) { + boolean needStream = false; + switch (stream.getKind()) { + case ROW_INDEX: + needStream = true; + break; + case BLOOM_FILTER: + case BLOOM_FILTER_UTF8: + needStream = (sargColumns != null) && (bloomFilterKinds[column] == stream.getKind()); + break; + default: + // PASS + break; + } + if (needStream) { + result.addOrMerge(offset, offset + stream.getLength(), true, false); + } + } + } + offset += stream.getLength(); + } + return result.get(); + } + + // TODO: see planIndexReading; this is not needed here. + private static boolean hadBadBloomFilters(TypeDescription.Category category, + WriterVersion version) { + switch(category) { + case STRING: + case CHAR: + case VARCHAR: + return !version.includes(WriterVersion.HIVE_12055); + case DECIMAL: + return true; + default: + return false; + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java index cdd58df..d2bb641 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.common.io.DataCache; import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch; import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.orc.CompressionCodec; import org.apache.orc.DataReader; import org.apache.orc.OrcProto; @@ -47,6 +48,9 @@ public interface Reader extends org.apache.hadoop.hive.ql.io.orc.Reader { EncodedReader encodedReader(Object fileKey, DataCache dataCache, DataReader dataReader, PoolFactory pf, IoTrace trace) throws IOException; + /** Gets the compression codec for the underlying ORC file. */ + CompressionCodec getCodec(); + /** The factory that can create (or return) the pools used by encoded reader. */ public interface PoolFactory { <T> Pool<T> createPool(int size, PoolObjectHelper<T> helper); http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java index d47ba6b..a916d58 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.io.DataCache; +import org.apache.orc.CompressionCodec; import org.apache.orc.DataReader; import org.apache.hadoop.hive.ql.io.orc.OrcFile.ReaderOptions; @@ -35,7 +36,13 @@ class ReaderImpl extends org.apache.hadoop.hive.ql.io.orc.ReaderImpl implements @Override public EncodedReader encodedReader(Object fileKey, DataCache dataCache, DataReader dataReader, PoolFactory pf, IoTrace trace) throws IOException { - return new EncodedReaderImpl(fileKey, types, - codec, bufferSize, rowIndexStride, dataCache, dataReader, pf, trace); + return new EncodedReaderImpl(fileKey, types, getSchema(), codec, getWriterVersion(), + bufferSize, rowIndexStride, dataCache, dataReader, pf, trace); } + + @Override + public CompressionCodec getCodec() { + return codec; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index 8af84dc..9e9f8cf 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -253,7 +253,7 @@ PREHOOK: Input: default@orc_ppd Stage-1 FILE SYSTEM COUNTERS: HDFS_BYTES_READ: 16673 HDFS_BYTES_WRITTEN: 104 - HDFS_READ_OPS: 5 + HDFS_READ_OPS: 6 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -263,6 +263,9 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 Stage-1 LLAP IO COUNTERS: + ALLOCATED_BYTES: 262144 + ALLOCATED_USED_BYTES: 26 + CACHE_MISS_BYTES: 24 METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 3 @@ -288,7 +291,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1344 + HDFS_BYTES_READ: 1055 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -300,11 +303,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 524288 - ALLOCATED_USED_BYTES: 269 - CACHE_MISS_BYTES: 249 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 + ALLOCATED_BYTES: 1048576 + ALLOCATED_USED_BYTES: 2732 + CACHE_HIT_BYTES: 24 + CACHE_MISS_BYTES: 1055 + METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 @@ -327,7 +330,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 22 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -352,7 +355,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 16 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -377,7 +380,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -402,7 +405,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -427,7 +430,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -452,7 +455,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -477,7 +480,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1697 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -502,7 +505,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -527,7 +530,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1713 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -552,7 +555,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -577,7 +580,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 50 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -602,7 +605,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 318 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -655,6 +658,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 830 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -675,6 +680,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 830 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -697,7 +704,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 6786 + HDFS_BYTES_READ: 5691 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -709,12 +716,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 786432 - ALLOCATED_USED_BYTES: 11299 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 3980 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 + ALLOCATED_BYTES: 1310720 + ALLOCATED_USED_BYTES: 13796 + CACHE_HIT_BYTES: 24 + CACHE_MISS_BYTES: 5691 + METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 @@ -737,7 +743,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -762,7 +768,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -801,7 +807,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -826,6 +832,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 1735 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -846,7 +854,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -871,7 +879,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -896,7 +904,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -921,7 +929,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 81 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -946,7 +954,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 74 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -971,7 +979,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -996,7 +1004,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 13 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1021,7 +1029,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1046,7 +1054,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 7 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1071,6 +1079,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 1735 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -1091,6 +1101,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 1735 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -1111,6 +1123,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 1735 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -1131,7 +1145,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1156,7 +1170,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1181,7 +1195,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4229 + CACHE_HIT_BYTES: 6770 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1206,7 +1220,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4229 + CACHE_HIT_BYTES: 6770 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index 4536cbb..7b52493 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -253,7 +253,7 @@ PREHOOK: Input: default@orc_ppd Stage-1 FILE SYSTEM COUNTERS: HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 104 - HDFS_READ_OPS: 6 + HDFS_READ_OPS: 7 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -263,9 +263,10 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2094 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 524288 - ALLOCATED_USED_BYTES: 269 - CACHE_MISS_BYTES: 249 + ALLOCATED_BYTES: 1310720 + ALLOCATED_USED_BYTES: 2758 + CACHE_HIT_BYTES: 0 + CACHE_MISS_BYTES: 1079 METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 @@ -289,7 +290,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2094 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index 77b7f5a..7e169f2 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -205,7 +205,7 @@ PREHOOK: Input: default@orc_ppd Stage-1 FILE SYSTEM COUNTERS: HDFS_BYTES_READ: 16673 HDFS_BYTES_WRITTEN: 104 - HDFS_READ_OPS: 5 + HDFS_READ_OPS: 6 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -215,6 +215,9 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 Stage-1 LLAP IO COUNTERS: + ALLOCATED_BYTES: 262144 + ALLOCATED_USED_BYTES: 26 + CACHE_MISS_BYTES: 24 METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 3 @@ -240,7 +243,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1344 + HDFS_BYTES_READ: 1055 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -252,11 +255,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 524288 - ALLOCATED_USED_BYTES: 269 - CACHE_MISS_BYTES: 249 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 + ALLOCATED_BYTES: 1048576 + ALLOCATED_USED_BYTES: 2732 + CACHE_HIT_BYTES: 24 + CACHE_MISS_BYTES: 1055 + METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 @@ -279,7 +282,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 22 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -304,7 +307,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 16 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -329,7 +332,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -354,7 +357,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -379,7 +382,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -404,7 +407,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -429,7 +432,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1697 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -454,7 +457,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -479,7 +482,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1713 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -504,7 +507,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -529,7 +532,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 50 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -554,7 +557,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 318 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -607,6 +610,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 830 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -627,6 +632,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 830 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -649,7 +656,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 6786 + HDFS_BYTES_READ: 5691 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -661,12 +668,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 786432 - ALLOCATED_USED_BYTES: 11299 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 3980 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 + ALLOCATED_BYTES: 1310720 + ALLOCATED_USED_BYTES: 13796 + CACHE_HIT_BYTES: 24 + CACHE_MISS_BYTES: 5691 + METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 @@ -689,7 +695,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -714,7 +720,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -753,7 +759,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -778,6 +784,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 1735 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -798,7 +806,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -823,7 +831,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -848,7 +856,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -873,7 +881,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 81 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -898,7 +906,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 74 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -923,7 +931,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -948,7 +956,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 13 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -973,7 +981,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -998,7 +1006,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 7 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1023,6 +1031,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 1735 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -1043,6 +1053,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 1735 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -1063,6 +1075,8 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 1735 + CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 0 @@ -1083,7 +1097,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1108,7 +1122,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1133,7 +1147,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4229 + CACHE_HIT_BYTES: 6770 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1158,7 +1172,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4229 + CACHE_HIT_BYTES: 6770 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1171,9 +1185,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 7718 + HDFS_BYTES_READ: 4912 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 4 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1183,12 +1197,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 262144 - ALLOCATED_USED_BYTES: 8400 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 4809 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 + ALLOCATED_BYTES: 524288 + ALLOCATED_USED_BYTES: 8527 + CACHE_HIT_BYTES: 24 + CACHE_MISS_BYTES: 4912 + METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 @@ -1199,25 +1212,27 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 1751 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4809 - CACHE_MISS_BYTES: 0 + ALLOCATED_BYTES: 262144 + ALLOCATED_USED_BYTES: 2376 + CACHE_HIT_BYTES: 4936 + CACHE_MISS_BYTES: 1751 METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 100 + SELECTED_ROWGROUPS: 1 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY @@ -1236,7 +1251,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4809 + CACHE_HIT_BYTES: 4936 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -1257,17 +1272,17 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4809 + CACHE_HIT_BYTES: 6687 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 100 + SELECTED_ROWGROUPS: 1 2 PREHOOK: query: create temporary table tmp_orcppd stored as orc
