Author: gunther Date: Fri Nov 7 07:40:01 2014 New Revision: 1637302 URL: http://svn.apache.org/r1637302 Log: HIVE-8732: ORC string statistics are not merged correctly (Owen O'Malley, reviewed by Prasanth J)
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java Modified: hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out hive/trunk/ql/src/test/resources/orc-file-dump.out hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out hive/trunk/ql/src/test/results/clientpositive/annotate_stats_part.q.out hive/trunk/ql/src/test/results/clientpositive/annotate_stats_table.q.out hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out hive/trunk/ql/src/test/results/clientpositive/tez/orc_analyze.q.out hive/trunk/ql/src/test/results/clientpositive/tez/vector_decimal_10_0.q.out hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out Modified: hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java (original) +++ hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java Fri Nov 7 07:40:01 2014 @@ -16333,14 +16333,32 @@ public final class OrcProto { // repeated uint32 version = 4 [packed = true]; /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ java.util.List<java.lang.Integer> getVersionList(); /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ int getVersionCount(); /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ int getVersion(int index); @@ -16354,6 +16372,28 @@ public final class OrcProto { */ long getMetadataLength(); + // optional uint32 writerVersion = 6; + /** + * <code>optional uint32 writerVersion = 6;</code> + * + * <pre> + * Version of the writer: + * 0 (or missing) = original + * 1 = HIVE-8732 fixed + * </pre> + */ + boolean hasWriterVersion(); + /** + * <code>optional uint32 writerVersion = 6;</code> + * + * <pre> + * Version of the writer: + * 0 (or missing) = original + * 1 = HIVE-8732 fixed + * </pre> + */ + int getWriterVersion(); + // optional string magic = 8000; /** * <code>optional string magic = 8000;</code> @@ -16483,8 +16523,13 @@ public final class OrcProto { metadataLength_ = input.readUInt64(); break; } - case 64002: { + case 48: { bitField0_ |= 0x00000010; + writerVersion_ = input.readUInt32(); + break; + } + case 64002: { + bitField0_ |= 0x00000020; magic_ = input.readBytes(); break; } @@ -16584,6 +16629,12 @@ public final class OrcProto { private java.util.List<java.lang.Integer> version_; /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public java.util.List<java.lang.Integer> getVersionList() { @@ -16591,12 +16642,24 @@ public final class OrcProto { } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public int getVersionCount() { return version_.size(); } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public int getVersion(int index) { return version_.get(index); @@ -16619,6 +16682,34 @@ public final class OrcProto { return metadataLength_; } + // optional uint32 writerVersion = 6; + public static final int WRITERVERSION_FIELD_NUMBER = 6; + private int writerVersion_; + /** + * <code>optional uint32 writerVersion = 6;</code> + * + * <pre> + * Version of the writer: + * 0 (or missing) = original + * 1 = HIVE-8732 fixed + * </pre> + */ + public boolean hasWriterVersion() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * <code>optional uint32 writerVersion = 6;</code> + * + * <pre> + * Version of the writer: + * 0 (or missing) = original + * 1 = HIVE-8732 fixed + * </pre> + */ + public int getWriterVersion() { + return writerVersion_; + } + // optional string magic = 8000; public static final int MAGIC_FIELD_NUMBER = 8000; private java.lang.Object magic_; @@ -16630,7 +16721,7 @@ public final class OrcProto { * </pre> */ public boolean hasMagic() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000020) == 0x00000020); } /** * <code>optional string magic = 8000;</code> @@ -16680,6 +16771,7 @@ public final class OrcProto { compressionBlockSize_ = 0L; version_ = java.util.Collections.emptyList(); metadataLength_ = 0L; + writerVersion_ = 0; magic_ = ""; } private byte memoizedIsInitialized = -1; @@ -16714,6 +16806,9 @@ public final class OrcProto { output.writeUInt64(5, metadataLength_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeUInt32(6, writerVersion_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { output.writeBytes(8000, getMagicBytes()); } getUnknownFields().writeTo(output); @@ -16757,6 +16852,10 @@ public final class OrcProto { } if (((bitField0_ & 0x00000010) == 0x00000010)) { size += com.google.protobuf.CodedOutputStream + .computeUInt32Size(6, writerVersion_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream .computeBytesSize(8000, getMagicBytes()); } size += getUnknownFields().getSerializedSize(); @@ -16889,8 +16988,10 @@ public final class OrcProto { bitField0_ = (bitField0_ & ~0x00000008); metadataLength_ = 0L; bitField0_ = (bitField0_ & ~0x00000010); - magic_ = ""; + writerVersion_ = 0; bitField0_ = (bitField0_ & ~0x00000020); + magic_ = ""; + bitField0_ = (bitField0_ & ~0x00000040); return this; } @@ -16943,6 +17044,10 @@ public final class OrcProto { if (((from_bitField0_ & 0x00000020) == 0x00000020)) { to_bitField0_ |= 0x00000010; } + result.writerVersion_ = writerVersion_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000020; + } result.magic_ = magic_; result.bitField0_ = to_bitField0_; onBuilt(); @@ -16982,8 +17087,11 @@ public final class OrcProto { if (other.hasMetadataLength()) { setMetadataLength(other.getMetadataLength()); } + if (other.hasWriterVersion()) { + setWriterVersion(other.getWriterVersion()); + } if (other.hasMagic()) { - bitField0_ |= 0x00000020; + bitField0_ |= 0x00000040; magic_ = other.magic_; onChanged(); } @@ -17126,6 +17234,12 @@ public final class OrcProto { } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public java.util.List<java.lang.Integer> getVersionList() { @@ -17133,18 +17247,36 @@ public final class OrcProto { } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public int getVersionCount() { return version_.size(); } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public int getVersion(int index) { return version_.get(index); } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public Builder setVersion( int index, int value) { @@ -17155,6 +17287,12 @@ public final class OrcProto { } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public Builder addVersion(int value) { ensureVersionIsMutable(); @@ -17164,6 +17302,12 @@ public final class OrcProto { } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public Builder addAllVersion( java.lang.Iterable<? extends java.lang.Integer> values) { @@ -17174,6 +17318,12 @@ public final class OrcProto { } /** * <code>repeated uint32 version = 4 [packed = true];</code> + * + * <pre> + * the version of the file format + * [0, 11] = Hive 0.11 + * [0, 12] = Hive 0.12 + * </pre> */ public Builder clearVersion() { version_ = java.util.Collections.emptyList(); @@ -17215,6 +17365,63 @@ public final class OrcProto { return this; } + // optional uint32 writerVersion = 6; + private int writerVersion_ ; + /** + * <code>optional uint32 writerVersion = 6;</code> + * + * <pre> + * Version of the writer: + * 0 (or missing) = original + * 1 = HIVE-8732 fixed + * </pre> + */ + public boolean hasWriterVersion() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * <code>optional uint32 writerVersion = 6;</code> + * + * <pre> + * Version of the writer: + * 0 (or missing) = original + * 1 = HIVE-8732 fixed + * </pre> + */ + public int getWriterVersion() { + return writerVersion_; + } + /** + * <code>optional uint32 writerVersion = 6;</code> + * + * <pre> + * Version of the writer: + * 0 (or missing) = original + * 1 = HIVE-8732 fixed + * </pre> + */ + public Builder setWriterVersion(int value) { + bitField0_ |= 0x00000020; + writerVersion_ = value; + onChanged(); + return this; + } + /** + * <code>optional uint32 writerVersion = 6;</code> + * + * <pre> + * Version of the writer: + * 0 (or missing) = original + * 1 = HIVE-8732 fixed + * </pre> + */ + public Builder clearWriterVersion() { + bitField0_ = (bitField0_ & ~0x00000020); + writerVersion_ = 0; + onChanged(); + return this; + } + // optional string magic = 8000; private java.lang.Object magic_ = ""; /** @@ -17225,7 +17432,7 @@ public final class OrcProto { * </pre> */ public boolean hasMagic() { - return ((bitField0_ & 0x00000020) == 0x00000020); + return ((bitField0_ & 0x00000040) == 0x00000040); } /** * <code>optional string magic = 8000;</code> @@ -17277,7 +17484,7 @@ public final class OrcProto { if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000020; + bitField0_ |= 0x00000040; magic_ = value; onChanged(); return this; @@ -17290,7 +17497,7 @@ public final class OrcProto { * </pre> */ public Builder clearMagic() { - bitField0_ = (bitField0_ & ~0x00000020); + bitField0_ = (bitField0_ & ~0x00000040); magic_ = getDefaultInstance().getMagic(); onChanged(); return this; @@ -17307,7 +17514,7 @@ public final class OrcProto { if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000020; + bitField0_ |= 0x00000040; magic_ = value; onChanged(); return this; @@ -17513,13 +17720,14 @@ public final class OrcProto { "em\022\024\n\014numberOfRows\030\006 \001(\004\022F\n\nstatistics\030\007" + " \003(\01322.org.apache.hadoop.hive.ql.io.orc." + "ColumnStatistics\022\026\n\016rowIndexStride\030\010 \001(\r" + - "\"\305\001\n\nPostScript\022\024\n\014footerLength\030\001 \001(\004\022F\n" + + "\"\334\001\n\nPostScript\022\024\n\014footerLength\030\001 \001(\004\022F\n" + "\013compression\030\002 \001(\01621.org.apache.hadoop.h" + "ive.ql.io.orc.CompressionKind\022\034\n\024compres" + "sionBlockSize\030\003 \001(\004\022\023\n\007version\030\004 \003(\rB\002\020\001" + - "\022\026\n\016metadataLength\030\005 \001(\004\022\016\n\005magic\030\300> \001(\t", - "*:\n\017CompressionKind\022\010\n\004NONE\020\000\022\010\n\004ZLIB\020\001\022" + - "\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003" + "\022\026\n\016metadataLength\030\005 \001(\004\022\025\n\rwriterVersio", + "n\030\006 \001(\r\022\016\n\005magic\030\300> \001(\t*:\n\017CompressionKi" + + "nd\022\010\n\004NONE\020\000\022\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZ" + + "O\020\003" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -17651,7 +17859,7 @@ public final class OrcProto { internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor, - new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", "Version", "MetadataLength", "Magic", }); + new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", "Version", "MetadataLength", "WriterVersion", "Magic", }); return null; } }; Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java Fri Nov 7 07:40:01 2014 @@ -394,7 +394,8 @@ class ColumnStatisticsImpl implements Co } else if (str.minimum != null) { if (minimum.compareTo(str.minimum) > 0) { minimum = new Text(str.getMinimum()); - } else if (maximum.compareTo(str.maximum) < 0) { + } + if (maximum.compareTo(str.maximum) < 0) { maximum = new Text(str.getMaximum()); } } @@ -563,7 +564,8 @@ class ColumnStatisticsImpl implements Co } else if (dec.minimum != null) { if (minimum.compareTo(dec.minimum) > 0) { minimum = dec.minimum; - } else if (maximum.compareTo(dec.maximum) < 0) { + } + if (maximum.compareTo(dec.maximum) < 0) { maximum = dec.maximum; } if (sum == null || dec.sum == null) { @@ -671,7 +673,8 @@ class ColumnStatisticsImpl implements Co } else if (dateStats.minimum != null) { if (minimum > dateStats.minimum) { minimum = dateStats.minimum; - } else if (maximum < dateStats.maximum) { + } + if (maximum < dateStats.maximum) { maximum = dateStats.maximum; } } @@ -767,7 +770,8 @@ class ColumnStatisticsImpl implements Co } else if (timestampStats.minimum != null) { if (minimum > timestampStats.minimum) { minimum = timestampStats.minimum; - } else if (maximum < timestampStats.maximum) { + } + if (maximum < timestampStats.maximum) { maximum = timestampStats.maximum; } } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java Fri Nov 7 07:40:01 2014 @@ -65,6 +65,8 @@ public final class FileDump { System.out.println("Structure for " + filename); Path path = new Path(filename); Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(conf)); + System.out.println("File Version: " + reader.getFileVersion().getName() + + " with " + reader.getWriterVersion()); RecordReaderImpl rows = (RecordReaderImpl) reader.rows(); System.out.println("Rows: " + reader.getNumberOfRows()); System.out.println("Compression: " + reader.getCompression()); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java Fri Nov 7 07:40:01 2014 @@ -97,6 +97,26 @@ public final class OrcFile { } } + /** + * Records the version of the writer in terms of which bugs have been fixed. + * For bugs in the writer, but the old readers already read the new data + * correctly, bump this version instead of the Version. + */ + public static enum WriterVersion { + ORIGINAL(0), + HIVE_8732(1); // corrupted stripe/file maximum column statistics + + private final int id; + + public int getId() { + return id; + } + + private WriterVersion(int id) { + this.id = id; + } + } + public static enum EncodingStrategy { SPEED, COMPRESSION; } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Fri Nov 7 07:40:01 2014 @@ -630,6 +630,7 @@ public class OrcInputFormat implements private final boolean isOriginal; private final List<Long> deltas; private final boolean hasBase; + private OrcFile.WriterVersion writerVersion; SplitGenerator(Context context, FileSystem fs, FileStatus file, FileInfo fileInfo, @@ -775,7 +776,9 @@ public class OrcInputFormat implements Reader.Options options = new Reader.Options(); setIncludedColumns(options, types, context.conf, isOriginal); setSearchArgument(options, types, context.conf, isOriginal); - if (options.getSearchArgument() != null) { + // only do split pruning if HIVE-8732 has been fixed in the writer + if (options.getSearchArgument() != null && + writerVersion != OrcFile.WriterVersion.ORIGINAL) { SearchArgument sarg = options.getSearchArgument(); List<PredicateLeaf> sargLeaves = sarg.getLeaves(); List<StripeStatistics> stripeStats = metadata.getStripeStatistics(); @@ -866,6 +869,7 @@ public class OrcInputFormat implements fileMetaInfo = fileInfo.fileMetaInfo; metadata = fileInfo.metadata; types = fileInfo.types; + writerVersion = fileInfo.writerVersion; // For multiple runs, in case sendSplitsInFooter changes if (fileMetaInfo == null && context.footerInSplits) { orcReader = OrcFile.createReader(file.getPath(), @@ -873,6 +877,7 @@ public class OrcInputFormat implements fileInfo.fileMetaInfo = ((ReaderImpl) orcReader).getFileMetaInfo(); fileInfo.metadata = orcReader.getMetadata(); fileInfo.types = orcReader.getTypes(); + fileInfo.writerVersion = orcReader.getWriterVersion(); } } else { orcReader = OrcFile.createReader(file.getPath(), @@ -880,13 +885,14 @@ public class OrcInputFormat implements stripes = orcReader.getStripes(); metadata = orcReader.getMetadata(); types = orcReader.getTypes(); + writerVersion = orcReader.getWriterVersion(); fileMetaInfo = context.footerInSplits ? ((ReaderImpl) orcReader).getFileMetaInfo() : null; if (context.cacheStripeDetails) { // Populate into cache. Context.footerCache.put(file.getPath(), new FileInfo(file.getModificationTime(), file.getLen(), stripes, - metadata, types, fileMetaInfo)); + metadata, types, fileMetaInfo, writerVersion)); } } } catch (Throwable th) { @@ -981,18 +987,21 @@ public class OrcInputFormat implements ReaderImpl.FileMetaInfo fileMetaInfo; Metadata metadata; List<OrcProto.Type> types; + private OrcFile.WriterVersion writerVersion; FileInfo(long modificationTime, long size, List<StripeInformation> stripeInfos, Metadata metadata, List<OrcProto.Type> types, - ReaderImpl.FileMetaInfo fileMetaInfo) { + ReaderImpl.FileMetaInfo fileMetaInfo, + OrcFile.WriterVersion writerVersion) { this.modificationTime = modificationTime; this.size = size; this.stripeInfos = stripeInfos; this.fileMetaInfo = fileMetaInfo; this.metadata = metadata; this.types = types; + this.writerVersion = writerVersion; } } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java Fri Nov 7 07:40:01 2014 @@ -38,6 +38,7 @@ public class OrcNewSplit extends FileSpl private boolean isOriginal; private boolean hasBase; private final List<Long> deltas = new ArrayList<Long>(); + private OrcFile.WriterVersion writerVersion; protected OrcNewSplit(){ //The FileSplit() constructor in hadoop 0.20 and 1.x is package private so can't use it. @@ -83,6 +84,7 @@ public class OrcNewSplit extends FileSpl WritableUtils.writeVInt(out, footerBuff.limit() - footerBuff.position()); out.write(footerBuff.array(), footerBuff.position(), footerBuff.limit() - footerBuff.position()); + WritableUtils.writeVInt(out, fileMetaInfo.writerVersion.getId()); } } @@ -111,9 +113,11 @@ public class OrcNewSplit extends FileSpl int footerBuffSize = WritableUtils.readVInt(in); ByteBuffer footerBuff = ByteBuffer.allocate(footerBuffSize); in.readFully(footerBuff.array(), 0, footerBuffSize); + OrcFile.WriterVersion writerVersion = + ReaderImpl.getWriterVersion(WritableUtils.readVInt(in)); fileMetaInfo = new ReaderImpl.FileMetaInfo(compressionType, bufferSize, - metadataSize, footerBuff); + metadataSize, footerBuff, writerVersion); } } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java Fri Nov 7 07:40:01 2014 @@ -42,6 +42,7 @@ public class OrcSplit extends FileSplit private boolean isOriginal; private boolean hasBase; private final List<Long> deltas = new ArrayList<Long>(); + private OrcFile.WriterVersion writerVersion; static final int BASE_FLAG = 4; static final int ORIGINAL_FLAG = 2; @@ -92,6 +93,7 @@ public class OrcSplit extends FileSplit WritableUtils.writeVInt(out, footerBuff.limit() - footerBuff.position()); out.write(footerBuff.array(), footerBuff.position(), footerBuff.limit() - footerBuff.position()); + WritableUtils.writeVInt(out, fileMetaInfo.writerVersion.getId()); } } @@ -120,9 +122,11 @@ public class OrcSplit extends FileSplit int footerBuffSize = WritableUtils.readVInt(in); ByteBuffer footerBuff = ByteBuffer.allocate(footerBuffSize); in.readFully(footerBuff.array(), 0, footerBuffSize); + OrcFile.WriterVersion writerVersion = + ReaderImpl.getWriterVersion(WritableUtils.readVInt(in)); fileMetaInfo = new ReaderImpl.FileMetaInfo(compressionType, bufferSize, - metadataSize, footerBuff); + metadataSize, footerBuff, writerVersion); } } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java Fri Nov 7 07:40:01 2014 @@ -129,6 +129,16 @@ public interface Reader { List<OrcProto.Type> getTypes(); /** + * Get the file format version. + */ + OrcFile.Version getFileVersion(); + + /** + * Get the version of the writer of this file. + */ + OrcFile.WriterVersion getWriterVersion(); + + /** * Options for creating a RecordReader. */ public static class Options { Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java Fri Nov 7 07:40:01 2014 @@ -62,6 +62,7 @@ final class ReaderImpl implements Reader private long deserializedSize = -1; private final Configuration conf; private final List<Integer> versionList; + private final OrcFile.WriterVersion writerVersion; //serialized footer - Keeping this around for use by getFileMetaInfo() // will help avoid cpu cycles spend in deserializing at cost of increased @@ -182,6 +183,22 @@ final class ReaderImpl implements Reader } @Override + public OrcFile.Version getFileVersion() { + for (OrcFile.Version version: OrcFile.Version.values()) { + if (version.getMajor() == versionList.get(0) && + version.getMinor() == versionList.get(1)) { + return version; + } + } + return OrcFile.Version.V_0_11; + } + + @Override + public OrcFile.WriterVersion getWriterVersion() { + return writerVersion; + } + + @Override public int getRowIndexStride() { return footer.getRowIndexStride(); } @@ -309,8 +326,22 @@ final class ReaderImpl implements Reader this.footer = rInfo.footer; this.inspector = rInfo.inspector; this.versionList = footerMetaData.versionList; + this.writerVersion = footerMetaData.writerVersion; } + /** + * Get the WriterVersion based on the ORC file postscript. + * @param writerVersion the integer writer version + * @return + */ + static OrcFile.WriterVersion getWriterVersion(int writerVersion) { + for(OrcFile.WriterVersion version: OrcFile.WriterVersion.values()) { + if (version.getId() == writerVersion) { + return version; + } + } + return OrcFile.WriterVersion.ORIGINAL; + } private static FileMetaInfo extractMetaInfoFromFooter(FileSystem fs, Path path, @@ -346,6 +377,12 @@ final class ReaderImpl implements Reader int footerSize = (int) ps.getFooterLength(); int metadataSize = (int) ps.getMetadataLength(); + OrcFile.WriterVersion writerVersion; + if (ps.hasWriterVersion()) { + writerVersion = getWriterVersion(ps.getWriterVersion()); + } else { + writerVersion = OrcFile.WriterVersion.ORIGINAL; + } //check compression codec switch (ps.getCompression()) { @@ -391,7 +428,8 @@ final class ReaderImpl implements Reader (int) ps.getCompressionBlockSize(), (int) ps.getMetadataLength(), buffer, - ps.getVersionList() + ps.getVersionList(), + writerVersion ); } @@ -451,25 +489,29 @@ final class ReaderImpl implements Reader final int metadataSize; final ByteBuffer footerBuffer; final List<Integer> versionList; + final OrcFile.WriterVersion writerVersion; FileMetaInfo(String compressionType, int bufferSize, int metadataSize, - ByteBuffer footerBuffer) { - this(compressionType, bufferSize, metadataSize, footerBuffer, null); + ByteBuffer footerBuffer, OrcFile.WriterVersion writerVersion) { + this(compressionType, bufferSize, metadataSize, footerBuffer, null, + writerVersion); } FileMetaInfo(String compressionType, int bufferSize, int metadataSize, - ByteBuffer footerBuffer, List<Integer> versionList){ + ByteBuffer footerBuffer, List<Integer> versionList, + OrcFile.WriterVersion writerVersion){ this.compressionType = compressionType; this.bufferSize = bufferSize; this.metadataSize = metadataSize; this.footerBuffer = footerBuffer; this.versionList = versionList; + this.writerVersion = writerVersion; } } public FileMetaInfo getFileMetaInfo(){ return new FileMetaInfo(compressionKind.toString(), bufferSize, - metadataSize, footerByteBuffer, versionList); + metadataSize, footerByteBuffer, versionList, writerVersion); } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Fri Nov 7 07:40:01 2014 @@ -2230,7 +2230,8 @@ class WriterImpl implements Writer, Memo .setMetadataLength(metadataLength) .setMagic(OrcFile.MAGIC) .addVersion(version.getMajor()) - .addVersion(version.getMinor()); + .addVersion(version.getMinor()) + .setWriterVersion(OrcFile.WriterVersion.HIVE_8732.getId()); if (compress != CompressionKind.NONE) { builder.setCompressionBlockSize(bufferSize); } Modified: hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original) +++ hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Fri Nov 7 07:40:01 2014 @@ -191,8 +191,15 @@ message PostScript { optional uint64 footerLength = 1; optional CompressionKind compression = 2; optional uint64 compressionBlockSize = 3; + // the version of the file format + // [0, 11] = Hive 0.11 + // [0, 12] = Hive 0.12 repeated uint32 version = 4 [packed = true]; optional uint64 metadataLength = 5; + // Version of the writer: + // 0 (or missing) = original + // 1 = HIVE-8732 fixed + optional uint32 writerVersion = 6; // Leave this last in the record optional string magic = 8000; } Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java?rev=1637302&view=auto ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java (added) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java Fri Nov 7 07:40:01 2014 @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import java.sql.Timestamp; + +import static junit.framework.Assert.assertEquals; + +/** + * Test ColumnStatisticsImpl for ORC. + */ +public class TestColumnStatistics { + + @Test + public void testLongMerge() throws Exception { + ObjectInspector inspector = + PrimitiveObjectInspectorFactory.javaIntObjectInspector; + + ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector); + ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector); + stats1.updateInteger(10); + stats1.updateInteger(10); + stats2.updateInteger(1); + stats2.updateInteger(1000); + stats1.merge(stats2); + IntegerColumnStatistics typed = (IntegerColumnStatistics) stats1; + assertEquals(1, typed.getMinimum()); + assertEquals(1000, typed.getMaximum()); + stats1.reset(); + stats1.updateInteger(-10); + stats1.updateInteger(10000); + stats1.merge(stats2); + assertEquals(-10, typed.getMinimum()); + assertEquals(10000, typed.getMaximum()); + } + + @Test + public void testDoubleMerge() throws Exception { + ObjectInspector inspector = + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector; + + ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector); + ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector); + stats1.updateDouble(10.0); + stats1.updateDouble(100.0); + stats2.updateDouble(1.0); + stats2.updateDouble(1000.0); + stats1.merge(stats2); + DoubleColumnStatistics typed = (DoubleColumnStatistics) stats1; + assertEquals(1.0, typed.getMinimum(), 0.001); + assertEquals(1000.0, typed.getMaximum(), 0.001); + stats1.reset(); + stats1.updateDouble(-10); + stats1.updateDouble(10000); + stats1.merge(stats2); + assertEquals(-10, typed.getMinimum(), 0.001); + assertEquals(10000, typed.getMaximum(), 0.001); + } + + + @Test + public void testStringMerge() throws Exception { + ObjectInspector inspector = + PrimitiveObjectInspectorFactory.javaStringObjectInspector; + + ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector); + ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector); + stats1.updateString(new Text("bob")); + stats1.updateString(new Text("david")); + stats1.updateString(new Text("charles")); + stats2.updateString(new Text("anne")); + stats2.updateString(new Text("erin")); + stats1.merge(stats2); + StringColumnStatistics typed = (StringColumnStatistics) stats1; + assertEquals("anne", typed.getMinimum()); + assertEquals("erin", typed.getMaximum()); + stats1.reset(); + stats1.updateString(new Text("aaa")); + stats1.updateString(new Text("zzz")); + stats1.merge(stats2); + assertEquals("aaa", typed.getMinimum()); + assertEquals("zzz", typed.getMaximum()); + } + + @Test + public void testDateMerge() throws Exception { + ObjectInspector inspector = + PrimitiveObjectInspectorFactory.javaDateObjectInspector; + + ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector); + ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector); + stats1.updateDate(new DateWritable(1000)); + stats1.updateDate(new DateWritable(100)); + stats2.updateDate(new DateWritable(10)); + stats2.updateDate(new DateWritable(2000)); + stats1.merge(stats2); + DateColumnStatistics typed = (DateColumnStatistics) stats1; + assertEquals(new DateWritable(10), typed.getMinimum()); + assertEquals(new DateWritable(2000), typed.getMaximum()); + stats1.reset(); + stats1.updateDate(new DateWritable(-10)); + stats1.updateDate(new DateWritable(10000)); + stats1.merge(stats2); + assertEquals(-10, typed.getMinimum().getDays()); + assertEquals(10000, typed.getMaximum().getDays()); + } + + @Test + public void testTimestampMerge() throws Exception { + ObjectInspector inspector = + PrimitiveObjectInspectorFactory.javaTimestampObjectInspector; + + ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector); + ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector); + stats1.updateTimestamp(new Timestamp(10)); + stats1.updateTimestamp(new Timestamp(100)); + stats2.updateTimestamp(new Timestamp(1)); + stats2.updateTimestamp(new Timestamp(1000)); + stats1.merge(stats2); + TimestampColumnStatistics typed = (TimestampColumnStatistics) stats1; + assertEquals(1, typed.getMinimum().getTime()); + assertEquals(1000, typed.getMaximum().getTime()); + stats1.reset(); + stats1.updateTimestamp(new Timestamp(-10)); + stats1.updateTimestamp(new Timestamp(10000)); + stats1.merge(stats2); + assertEquals(-10, typed.getMinimum().getTime()); + assertEquals(10000, typed.getMaximum().getTime()); + } + + @Test + public void testDecimalMerge() throws Exception { + ObjectInspector inspector = + PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector; + + ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector); + ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector); + stats1.updateDecimal(HiveDecimal.create(10)); + stats1.updateDecimal(HiveDecimal.create(100)); + stats2.updateDecimal(HiveDecimal.create(1)); + stats2.updateDecimal(HiveDecimal.create(1000)); + stats1.merge(stats2); + DecimalColumnStatistics typed = (DecimalColumnStatistics) stats1; + assertEquals(1, typed.getMinimum().longValue()); + assertEquals(1000, typed.getMaximum().longValue()); + stats1.reset(); + stats1.updateDecimal(HiveDecimal.create(-10)); + stats1.updateDecimal(HiveDecimal.create(10000)); + stats1.merge(stats2); + assertEquals(-10, typed.getMinimum().longValue()); + assertEquals(10000, typed.getMaximum().longValue()); + } +} Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Fri Nov 7 07:40:01 2014 @@ -1620,14 +1620,14 @@ public class TestInputOutputFormat { assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00000", split.getPath().toString()); assertEquals(0, split.getStart()); - assertEquals(580, split.getLength()); + assertEquals(582, split.getLength()); split = (HiveInputFormat.HiveInputSplit) splits[1]; assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", split.inputFormatClassName()); assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00001", split.getPath().toString()); assertEquals(0, split.getStart()); - assertEquals(601, split.getLength()); + assertEquals(603, split.getLength()); CombineHiveInputFormat.CombineHiveInputSplit combineSplit = (CombineHiveInputFormat.CombineHiveInputSplit) splits[2]; assertEquals(BUCKETS, combineSplit.getNumPaths()); @@ -1635,7 +1635,7 @@ public class TestInputOutputFormat { assertEquals("mock:/combinationAcid/p=1/00000" + bucket + "_0", combineSplit.getPath(bucket).toString()); assertEquals(0, combineSplit.getOffset(bucket)); - assertEquals(225, combineSplit.getLength(bucket)); + assertEquals(227, combineSplit.getLength(bucket)); } String[] hosts = combineSplit.getLocations(); assertEquals(2, hosts.length); Modified: hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out (original) +++ hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out Fri Nov 7 07:40:01 2014 @@ -1,4 +1,5 @@ Structure for TestFileDump.testDump.orc +File Version: 0.12 with HIVE_8732 Rows: 21000 Compression: ZLIB Compression size: 10000 @@ -182,6 +183,6 @@ Stripes: Row group index column 3: Entry 0: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214-18444-18446-18724-18912-18952-19164-19348-19400-19546-19776-19896-20084 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952- 7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 positions: 0,0,0,0,0 -File length: 2033557 bytes +File length: 2033559 bytes Padding length: 0 bytes Padding ratio: 0% Modified: hive/trunk/ql/src/test/resources/orc-file-dump.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/resources/orc-file-dump.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/resources/orc-file-dump.out (original) +++ hive/trunk/ql/src/test/resources/orc-file-dump.out Fri Nov 7 07:40:01 2014 @@ -1,4 +1,5 @@ Structure for TestFileDump.testDump.orc +File Version: 0.12 with HIVE_8732 Rows: 21000 Compression: ZLIB Compression size: 10000 @@ -187,6 +188,6 @@ Stripes: Row group index column 3: Entry 0: count: 1000 min: Darkness, max: worst positions: 0,0,0 -File length: 270760 bytes +File length: 270762 bytes Padding length: 0 bytes Padding ratio: 0% Modified: hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out Fri Nov 7 07:40:01 2014 @@ -48,9 +48,9 @@ columns:struct columns { i32 key, string partitioned:false partitionColumns: totalNumberFiles:3 -totalFileSize:7488 -maxFileSize:2496 -minFileSize:2496 +totalFileSize:7494 +maxFileSize:2498 +minFileSize:2498 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test @@ -91,9 +91,9 @@ columns:struct columns { i32 key, string partitioned:false partitionColumns: totalNumberFiles:1 -totalFileSize:7167 -maxFileSize:7167 -minFileSize:7167 +totalFileSize:7169 +maxFileSize:7169 +minFileSize:7169 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test @@ -171,9 +171,9 @@ columns:struct columns { i32 key, string partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:3 -totalFileSize:7488 -maxFileSize:2496 -minFileSize:2496 +totalFileSize:7494 +maxFileSize:2498 +minFileSize:2498 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test_part @@ -218,9 +218,9 @@ columns:struct columns { i32 key, string partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:1 -totalFileSize:7167 -maxFileSize:7167 -minFileSize:7167 +totalFileSize:7169 +maxFileSize:7169 +minFileSize:7169 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test_part Modified: hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out Fri Nov 7 07:40:01 2014 @@ -48,9 +48,9 @@ columns:struct columns { i32 key, string partitioned:false partitionColumns: totalNumberFiles:3 -totalFileSize:7488 -maxFileSize:2496 -minFileSize:2496 +totalFileSize:7494 +maxFileSize:2498 +minFileSize:2498 #### A masked pattern was here #### PREHOOK: query: desc extended src_orc_merge_test_stat @@ -94,7 +94,7 @@ Table Parameters: numFiles 3 numRows 1500 rawDataSize 141000 - totalSize 7488 + totalSize 7494 #### A masked pattern was here #### # Storage Information @@ -146,7 +146,7 @@ Table Parameters: numFiles 1 numRows 1500 rawDataSize 141000 - totalSize 7167 + totalSize 7169 #### A masked pattern was here #### # Storage Information @@ -216,9 +216,9 @@ columns:struct columns { i32 key, string partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:3 -totalFileSize:7488 -maxFileSize:2496 -minFileSize:2496 +totalFileSize:7494 +maxFileSize:2498 +minFileSize:2498 #### A masked pattern was here #### PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') @@ -249,7 +249,7 @@ Partition Parameters: numFiles 3 numRows 1500 rawDataSize 141000 - totalSize 7488 + totalSize 7494 #### A masked pattern was here #### # Storage Information @@ -300,7 +300,7 @@ Partition Parameters: numFiles 3 numRows 1500 rawDataSize 141000 - totalSize 7488 + totalSize 7494 #### A masked pattern was here #### # Storage Information @@ -359,7 +359,7 @@ Partition Parameters: numFiles 1 numRows 1500 rawDataSize 141000 - totalSize 7167 + totalSize 7169 #### A masked pattern was here #### # Storage Information Modified: hive/trunk/ql/src/test/results/clientpositive/annotate_stats_part.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/annotate_stats_part.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/annotate_stats_part.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/annotate_stats_part.q.out Fri Nov 7 07:40:01 2014 @@ -98,11 +98,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 5 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- partition level analyze statistics for specific parition @@ -135,11 +135,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 325 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 325 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- basicStatState: PARTIAL colStatState: NONE @@ -158,11 +158,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 9 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 9 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE @@ -181,11 +181,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), '2001' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- partition level analyze statistics for all partitions @@ -222,11 +222,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 325 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 325 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE @@ -245,11 +245,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE @@ -268,11 +268,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- both partitions will be pruned @@ -331,11 +331,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: zip (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: PARTIAL @@ -354,7 +354,7 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string) outputColumnNames: _col0 @@ -377,7 +377,7 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: year (type: string) outputColumnNames: _col0 @@ -402,7 +402,7 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int) outputColumnNames: _col0, _col1 @@ -425,7 +425,7 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), locid (type: int) outputColumnNames: _col0, _col1 @@ -448,11 +448,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 325 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: state (type: string), locid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 325 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: PARTIAL @@ -471,11 +471,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 727 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- This is to test filter expression evaluation on partition column @@ -496,7 +496,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: loc_orc - Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -532,7 +532,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: loc_orc - Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -568,7 +568,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: loc_orc - Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Modified: hive/trunk/ql/src/test/results/clientpositive/annotate_stats_table.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/annotate_stats_table.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/annotate_stats_table.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/annotate_stats_table.q.out Fri Nov 7 07:40:01 2014 @@ -89,11 +89,11 @@ STAGE PLANS: Processor Tree: TableScan alias: emp_orc - Statistics: Num rows: 3 Data size: 364 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 366 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: lastname (type: string), deptid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 364 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 366 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- table level analyze statistics @@ -122,11 +122,11 @@ STAGE PLANS: Processor Tree: TableScan alias: emp_orc - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: lastname (type: string), deptid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- column level partial statistics @@ -155,11 +155,11 @@ STAGE PLANS: Processor Tree: TableScan alias: emp_orc - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: lastname (type: string), deptid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- all selected columns have statistics @@ -180,7 +180,7 @@ STAGE PLANS: Processor Tree: TableScan alias: emp_orc - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: deptid (type: int) outputColumnNames: _col0 @@ -213,11 +213,11 @@ STAGE PLANS: Processor Tree: TableScan alias: emp_orc - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: lastname (type: string), deptid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: COMPLETE @@ -236,7 +236,7 @@ STAGE PLANS: Processor Tree: TableScan alias: emp_orc - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: lastname (type: string) outputColumnNames: _col0 @@ -259,7 +259,7 @@ STAGE PLANS: Processor Tree: TableScan alias: emp_orc - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: deptid (type: int) outputColumnNames: _col0 @@ -282,7 +282,7 @@ STAGE PLANS: Processor Tree: TableScan alias: emp_orc - Statistics: Num rows: 48 Data size: 364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 48 Data size: 366 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: lastname (type: string), deptid (type: int) outputColumnNames: _col0, _col1 Modified: hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out Fri Nov 7 07:40:01 2014 @@ -880,7 +880,7 @@ Partition Parameters: numFiles 2 numRows 32 rawDataSize 640 - totalSize 1348 + totalSize 1352 #### A masked pattern was here #### # Storage Information @@ -924,7 +924,7 @@ Partition Parameters: numFiles 2 numRows 6 rawDataSize 120 - totalSize 1050 + totalSize 1054 #### A masked pattern was here #### # Storage Information @@ -968,7 +968,7 @@ Partition Parameters: numFiles 2 numRows 14 rawDataSize 280 - totalSize 1166 + totalSize 1170 #### A masked pattern was here #### # Storage Information @@ -1012,7 +1012,7 @@ Partition Parameters: numFiles 2 numRows 6 rawDataSize 120 - totalSize 1050 + totalSize 1054 #### A masked pattern was here #### # Storage Information @@ -1055,7 +1055,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4340 + totalSize 4356 #### A masked pattern was here #### # Storage Information @@ -1098,7 +1098,7 @@ Partition Parameters: numFiles 8 numRows 6 rawDataSize 120 - totalSize 2094 + totalSize 2110 #### A masked pattern was here #### # Storage Information @@ -1141,7 +1141,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4326 + totalSize 4342 #### A masked pattern was here #### # Storage Information @@ -1184,7 +1184,7 @@ Partition Parameters: numFiles 8 numRows 6 rawDataSize 120 - totalSize 2094 + totalSize 2110 #### A masked pattern was here #### # Storage Information Modified: hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out Fri Nov 7 07:40:01 2014 @@ -1183,7 +1183,7 @@ Partition Parameters: numFiles 1 numRows 11 rawDataSize 88 - totalSize 417 + totalSize 419 #### A masked pattern was here #### # Storage Information @@ -1245,7 +1245,7 @@ Partition Parameters: numFiles 1 numRows 13 rawDataSize 104 - totalSize 440 + totalSize 442 #### A masked pattern was here #### # Storage Information @@ -1404,7 +1404,7 @@ Partition Parameters: numFiles 1 numRows 11 rawDataSize 88 - totalSize 417 + totalSize 419 #### A masked pattern was here #### # Storage Information @@ -1466,7 +1466,7 @@ Partition Parameters: numFiles 1 numRows 13 rawDataSize 104 - totalSize 440 + totalSize 442 #### A masked pattern was here #### # Storage Information Modified: hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== Files hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out (original) and hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out Fri Nov 7 07:40:01 2014 differ Modified: hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== Files hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out (original) and hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out Fri Nov 7 07:40:01 2014 differ Modified: hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out Fri Nov 7 07:40:01 2014 @@ -107,7 +107,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 52600 - totalSize 3121 + totalSize 3123 #### A masked pattern was here #### # Storage Information @@ -197,7 +197,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 52600 - totalSize 3121 + totalSize 3123 #### A masked pattern was here #### # Storage Information @@ -313,7 +313,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 21950 - totalSize 2024 + totalSize 2026 #### A masked pattern was here #### # Storage Information @@ -358,7 +358,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2043 + totalSize 2045 #### A masked pattern was here #### # Storage Information @@ -460,7 +460,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 21950 - totalSize 2024 + totalSize 2026 #### A masked pattern was here #### # Storage Information @@ -505,7 +505,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2043 + totalSize 2045 #### A masked pattern was here #### # Storage Information @@ -627,7 +627,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 21950 - totalSize 2024 + totalSize 2026 #### A masked pattern was here #### # Storage Information @@ -672,7 +672,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2043 + totalSize 2045 #### A masked pattern was here #### # Storage Information @@ -780,7 +780,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 21950 - totalSize 2024 + totalSize 2026 #### A masked pattern was here #### # Storage Information @@ -825,7 +825,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2043 + totalSize 2045 #### A masked pattern was here #### # Storage Information @@ -992,7 +992,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 21950 - totalSize 2024 + totalSize 2026 #### A masked pattern was here #### # Storage Information Modified: hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out Fri Nov 7 07:40:01 2014 @@ -48,9 +48,9 @@ columns:struct columns { i32 key, string partitioned:false partitionColumns: totalNumberFiles:3 -totalFileSize:7488 -maxFileSize:2496 -minFileSize:2496 +totalFileSize:7494 +maxFileSize:2498 +minFileSize:2498 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test @@ -91,9 +91,9 @@ columns:struct columns { i32 key, string partitioned:false partitionColumns: totalNumberFiles:1 -totalFileSize:7167 -maxFileSize:7167 -minFileSize:7167 +totalFileSize:7169 +maxFileSize:7169 +minFileSize:7169 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test @@ -171,9 +171,9 @@ columns:struct columns { i32 key, string partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:3 -totalFileSize:7488 -maxFileSize:2496 -minFileSize:2496 +totalFileSize:7494 +maxFileSize:2498 +minFileSize:2498 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test_part @@ -218,9 +218,9 @@ columns:struct columns { i32 key, string partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:1 -totalFileSize:7167 -maxFileSize:7167 -minFileSize:7167 +totalFileSize:7169 +maxFileSize:7169 +minFileSize:7169 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test_part Modified: hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out Fri Nov 7 07:40:01 2014 @@ -48,9 +48,9 @@ columns:struct columns { i32 key, string partitioned:false partitionColumns: totalNumberFiles:3 -totalFileSize:7488 -maxFileSize:2496 -minFileSize:2496 +totalFileSize:7494 +maxFileSize:2498 +minFileSize:2498 #### A masked pattern was here #### PREHOOK: query: desc extended src_orc_merge_test_stat @@ -94,7 +94,7 @@ Table Parameters: numFiles 3 numRows 1500 rawDataSize 141000 - totalSize 7488 + totalSize 7494 #### A masked pattern was here #### # Storage Information @@ -146,7 +146,7 @@ Table Parameters: numFiles 1 numRows 1500 rawDataSize 141000 - totalSize 7167 + totalSize 7169 #### A masked pattern was here #### # Storage Information @@ -216,9 +216,9 @@ columns:struct columns { i32 key, string partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:3 -totalFileSize:7488 -maxFileSize:2496 -minFileSize:2496 +totalFileSize:7494 +maxFileSize:2498 +minFileSize:2498 #### A masked pattern was here #### PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') @@ -249,7 +249,7 @@ Partition Parameters: numFiles 3 numRows 1500 rawDataSize 141000 - totalSize 7488 + totalSize 7494 #### A masked pattern was here #### # Storage Information @@ -300,7 +300,7 @@ Partition Parameters: numFiles 3 numRows 1500 rawDataSize 141000 - totalSize 7488 + totalSize 7494 #### A masked pattern was here #### # Storage Information @@ -359,7 +359,7 @@ Partition Parameters: numFiles 1 numRows 1500 rawDataSize 141000 - totalSize 7167 + totalSize 7169 #### A masked pattern was here #### # Storage Information Modified: hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out?rev=1637302&r1=1637301&r2=1637302&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out Fri Nov 7 07:40:01 2014 @@ -932,7 +932,7 @@ Partition Parameters: numFiles 2 numRows 32 rawDataSize 640 - totalSize 1348 + totalSize 1352 #### A masked pattern was here #### # Storage Information @@ -976,7 +976,7 @@ Partition Parameters: numFiles 2 numRows 6 rawDataSize 120 - totalSize 1050 + totalSize 1054 #### A masked pattern was here #### # Storage Information @@ -1020,7 +1020,7 @@ Partition Parameters: numFiles 2 numRows 14 rawDataSize 280 - totalSize 1166 + totalSize 1170 #### A masked pattern was here #### # Storage Information @@ -1064,7 +1064,7 @@ Partition Parameters: numFiles 2 numRows 6 rawDataSize 120 - totalSize 1050 + totalSize 1054 #### A masked pattern was here #### # Storage Information @@ -1107,7 +1107,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4340 + totalSize 4356 #### A masked pattern was here #### # Storage Information @@ -1150,7 +1150,7 @@ Partition Parameters: numFiles 8 numRows 6 rawDataSize 120 - totalSize 2094 + totalSize 2110 #### A masked pattern was here #### # Storage Information @@ -1193,7 +1193,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4326 + totalSize 4342 #### A masked pattern was here #### # Storage Information @@ -1236,7 +1236,7 @@ Partition Parameters: numFiles 8 numRows 6 rawDataSize 120 - totalSize 2094 + totalSize 2110 #### A masked pattern was here #### # Storage Information