Repository: hive Updated Branches: refs/heads/master 4a4ae12ca -> 50fb6f3cb
http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out index b799527..1157edb 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out @@ -215,9 +215,10 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 524288 - ALLOCATED_USED_BYTES: 269 - CACHE_MISS_BYTES: 249 + ALLOCATED_BYTES: 1048576 + ALLOCATED_USED_BYTES: 382 + CACHE_HIT_BYTES: 0 + CACHE_MISS_BYTES: 359 METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 @@ -255,7 +256,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -268,9 +269,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 720 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -280,8 +281,10 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 + ALLOCATED_BYTES: 262144 + ALLOCATED_USED_BYTES: 2376 + CACHE_HIT_BYTES: 359 + CACHE_MISS_BYTES: 720 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -305,7 +308,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -330,7 +333,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -359,7 +362,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -398,7 +401,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -423,7 +426,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -448,7 +451,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -473,7 +476,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -502,7 +505,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -541,7 +544,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -566,7 +569,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -591,7 +594,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -616,7 +619,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -645,7 +648,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -684,7 +687,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -709,7 +712,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -734,7 +737,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -759,7 +762,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 + CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -890,9 +893,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 5287 + HDFS_BYTES_READ: 4912 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 4 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -902,12 +905,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 262144 - ALLOCATED_USED_BYTES: 8400 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 4809 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 + ALLOCATED_BYTES: 524288 + ALLOCATED_USED_BYTES: 8527 + CACHE_HIT_BYTES: 24 + CACHE_MISS_BYTES: 4912 + METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 @@ -918,25 +920,27 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 1751 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4809 - CACHE_MISS_BYTES: 0 + ALLOCATED_BYTES: 262144 + ALLOCATED_USED_BYTES: 2376 + CACHE_HIT_BYTES: 4936 + CACHE_MISS_BYTES: 1751 METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 100 + SELECTED_ROWGROUPS: 1 2 PREHOOK: query: alter table orc_ppd change column f f double PREHOOK: type: ALTERTABLE_RENAMECOL @@ -1027,9 +1031,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 4577 + HDFS_BYTES_READ: 4099 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 4 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1039,12 +1043,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 786432 - ALLOCATED_USED_BYTES: 11299 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 3980 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 + ALLOCATED_BYTES: 1048576 + ALLOCATED_USED_BYTES: 11420 + CACHE_HIT_BYTES: 24 + CACHE_MISS_BYTES: 4099 + METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 @@ -1055,9 +1058,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 1592 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1067,8 +1070,10 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 + ALLOCATED_BYTES: 262144 + ALLOCATED_USED_BYTES: 2376 + CACHE_HIT_BYTES: 4123 + CACHE_MISS_BYTES: 1592 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 @@ -1138,7 +1143,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 4123 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -1163,7 +1168,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -1234,7 +1239,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 4123 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -1259,7 +1264,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 + CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -1276,9 +1281,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 2780 + HDFS_BYTES_READ: 2183 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 4 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1288,12 +1293,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 524288 - ALLOCATED_USED_BYTES: 4201 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 2098 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 + ALLOCATED_BYTES: 786432 + ALLOCATED_USED_BYTES: 4293 + CACHE_HIT_BYTES: 24 + CACHE_MISS_BYTES: 2183 + METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 @@ -1323,25 +1327,27 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 1217 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 2098 - CACHE_MISS_BYTES: 0 + ALLOCATED_BYTES: 262144 + ALLOCATED_USED_BYTES: 2376 + CACHE_HIT_BYTES: 2207 + CACHE_MISS_BYTES: 1217 METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 1000 + SELECTED_ROWGROUPS: 1 4 PREHOOK: query: select count(*) from orc_ppd where si = 442 or boo is not null or boo = false PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java index 403c3ad..e684ece 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java +++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.common.io; + +import java.nio.ByteBuffer; import java.io.IOException; import java.io.InputStream; @@ -30,8 +32,6 @@ public interface FileMetadataCache { */ MemoryBufferOrBuffers getFileMetadata(Object fileKey); - // TODO: add BB put method(s) when merging with ORC off-heap metadata cache - /** * Puts the metadata for a given file (e.g. a footer buffer into cache). * @param fileKey The file key. @@ -43,9 +43,11 @@ public interface FileMetadataCache { MemoryBufferOrBuffers putFileMetadata( Object fileKey, int length, InputStream is) throws IOException; + MemoryBufferOrBuffers putFileMetadata(Object fileKey, ByteBuffer tailBuffer); + /** * Releases the buffer returned from getFileMetadata or putFileMetadata method. * @param buffer The buffer to release. */ void decRefBuffer(MemoryBufferOrBuffers buffer); -} \ No newline at end of file +} \ No newline at end of file
