Repository: hive Updated Branches: refs/heads/master 6c4d7a9b9 -> aea590834
HIVE-15190: Field names are not preserved in ORC files written with ACID (Anthony Hsu reviewed by Prasanth Jayachandran) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aea59083 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aea59083 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aea59083 Branch: refs/heads/master Commit: aea5908341c7b7084551216efc6c89d64b1cb06a Parents: 6c4d7a9 Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Wed Jun 20 10:24:36 2018 -0700 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Wed Jun 20 10:24:41 2018 -0700 ---------------------------------------------------------------------- .../hive/ql/txn/compactor/TestCompactor.java | 2 +- .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 6 +- .../hive/ql/io/orc/OrcRawRecordMerger.java | 12 +++- .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 52 ++++++++++++++++- .../io/orc/VectorizedOrcAcidRowBatchReader.java | 2 +- .../apache/hadoop/hive/ql/TestTxnCommands2.java | 60 ++++++++++++++++++++ .../results/clientpositive/acid_nullscan.q.out | 8 +-- .../clientpositive/acid_table_stats.q.out | 14 ++--- .../clientpositive/autoColumnStats_4.q.out | 4 +- .../llap/acid_bucket_pruning.q.out | 6 +- .../llap/acid_vectorization_original.q.out | 4 +- .../llap/default_constraint.q.out | 14 ++--- .../llap/dynpart_sort_optimization_acid.q.out | 14 ++--- .../insert_values_orig_table_use_metadata.q.out | 24 ++++---- .../materialized_view_create_rewrite_3.q.out | 20 +++---- .../materialized_view_create_rewrite_4.q.out | 22 +++---- .../materialized_view_create_rewrite_5.q.out | 34 +++++------ ...ized_view_create_rewrite_rebuild_dummy.q.out | 20 +++---- .../test/results/clientpositive/row__id.q.out | 18 +++--- .../tez/acid_vectorization_original_tez.q.out | 4 +- .../clientpositive/tez/explainanalyze_5.q.out | 2 +- 21 files changed, 228 insertions(+), 114 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index 4a0e834..cffa21a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -1596,7 +1596,7 @@ public class TestCompactor { "'transactional'='true'," + "'compactor.mapreduce.map.memory.mb'='2048'," + // 2048 MB memory for compaction map job "'compactorthreshold.hive.compactor.delta.num.threshold'='4'," + // minor compaction if more than 4 delta dirs - "'compactorthreshold.hive.compactor.delta.pct.threshold'='0.49'" + // major compaction if more than 49% + "'compactorthreshold.hive.compactor.delta.pct.threshold'='0.47'" + // major compaction if more than 47% ")", driver); // Insert 5 rows to both tables http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java index 68e88cf..56d590c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java @@ -125,8 +125,10 @@ public final class OrcFile extends org.apache.orc.OrcFile { * @return this */ public WriterOptions setSchema(TypeDescription schema) { - this.explicitSchema = true; - super.setSchema(schema); + if (schema != null) { + this.explicitSchema = true; + super.setSchema(schema); + } return this; } http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java index 6571a24..929ea9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.orc.OrcUtils; import org.apache.orc.StripeInformation; import org.apache.orc.TypeDescription; +import org.apache.orc.impl.SchemaEvolution; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -773,9 +774,10 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{ * Convert from the row include/sarg/columnNames to the event equivalent * for the underlying file. * @param options options for the row reader + * @param rowSchema schema of the row, excluding ACID columns * @return a cloned options object that is modified for the event reader */ - static Reader.Options createEventOptions(Reader.Options options) { + static Reader.Options createEventOptions(Reader.Options options, TypeDescription rowSchema) { Reader.Options result = options.clone(); result.include(options.getInclude()); @@ -788,6 +790,10 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{ } result.searchArgument(options.getSearchArgument(), cols); } + + // schema evolution will insert the acid columns to row schema for ACID read + result.schema(rowSchema); + return result; } @@ -970,12 +976,12 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{ TypeDescription typeDescr = OrcInputFormat.getDesiredRowTypeDescr(conf, true, Integer.MAX_VALUE); - objectInspector = OrcRecordUpdater.createEventSchema + objectInspector = OrcRecordUpdater.createEventObjectInspector (OrcStruct.createObjectInspector(0, OrcUtils.getOrcTypes(typeDescr))); assert !(mergerOptions.isCompacting() && reader != null) : "don't need a reader for compaction"; // modify the options to reflect the event instead of the base row - Reader.Options eventOptions = createEventOptions(options); + Reader.Options eventOptions = createEventOptions(options, typeDescr); //suppose it's the first Major compaction so we only have deltas boolean isMajorNoBase = mergerOptions.isCompacting() && mergerOptions.isMajorCompaction() && mergerOptions.getBaseDir() == null; http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java index 1359dc3..5590470 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java @@ -23,7 +23,9 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -33,6 +35,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.AcidOutputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.BucketCodec; +import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.ql.io.RecordUpdater; import org.apache.hadoop.hive.serde2.SerDeStats; @@ -42,15 +45,21 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.orc.OrcConf; +import org.apache.orc.TypeDescription; import org.apache.orc.impl.AcidStats; import org.apache.orc.impl.OrcAcidUtils; +import org.apache.orc.impl.SchemaEvolution; import org.apache.orc.impl.WriterImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Strings; + /** * A RecordUpdater where the files are stored as ORC. * A note on various record structures: the {@code row} coming in (as in {@link #insert(long, Object)} @@ -179,7 +188,7 @@ public class OrcRecordUpdater implements RecordUpdater { * @param rowInspector the row's object inspector * @return an object inspector for the event stream */ - static StructObjectInspector createEventSchema(ObjectInspector rowInspector) { + static StructObjectInspector createEventObjectInspector(ObjectInspector rowInspector) { List<StructField> fields = new ArrayList<StructField>(); fields.add(new OrcStruct.Field("operation", PrimitiveObjectInspectorFactory.writableIntObjectInspector, OPERATION)); @@ -194,6 +203,41 @@ public class OrcRecordUpdater implements RecordUpdater { fields.add(new OrcStruct.Field("row", rowInspector, ROW)); return new OrcStruct.OrcStructInspector(fields); } + + private static TypeDescription createEventSchemaFromTableProperties(Properties tableProps) { + TypeDescription rowSchema = getTypeDescriptionFromTableProperties(tableProps); + if (rowSchema == null) { + return null; + } + + return SchemaEvolution.createEventSchema(rowSchema); + } + + private static TypeDescription getTypeDescriptionFromTableProperties(Properties tableProperties) { + TypeDescription schema = null; + if (tableProperties != null) { + final String columnNameProperty = tableProperties.getProperty(IOConstants.COLUMNS); + final String columnTypeProperty = tableProperties.getProperty(IOConstants.COLUMNS_TYPES); + if (!Strings.isNullOrEmpty(columnNameProperty) && !Strings.isNullOrEmpty(columnTypeProperty)) { + List<String> columnNames = + columnNameProperty.length() == 0 ? new ArrayList<String>() : Arrays.asList(columnNameProperty.split(",")); + List<TypeInfo> columnTypes = columnTypeProperty.length() == 0 ? new ArrayList<TypeInfo>() : TypeInfoUtils + .getTypeInfosFromTypeString(columnTypeProperty); + + schema = TypeDescription.createStruct(); + for (int i = 0; i < columnNames.size(); i++) { + schema.addField(columnNames.get(i), OrcInputFormat.convertTypeInfo(columnTypes.get(i))); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("ORC schema = " + schema); + } + + return schema; + } + /** * @param partitionRoot - partition root (or table root if not partitioned) */ @@ -282,8 +326,9 @@ public class OrcRecordUpdater implements RecordUpdater { */ this.deleteWriterOptions = OrcFile.writerOptions(optionsCloneForDelta.getTableProperties(), optionsCloneForDelta.getConfiguration()); - this.deleteWriterOptions.inspector(createEventSchema(findRecId(options.getInspector(), + this.deleteWriterOptions.inspector(createEventObjectInspector(findRecId(options.getInspector(), options.getRecordIdColumn()))); + this.deleteWriterOptions.setSchema(createEventSchemaFromTableProperties(options.getTableProperties())); } // get buffer size and stripe size for base writer @@ -305,8 +350,9 @@ public class OrcRecordUpdater implements RecordUpdater { } writerOptions.fileSystem(fs).callback(indexBuilder); rowInspector = (StructObjectInspector)options.getInspector(); - writerOptions.inspector(createEventSchema(findRecId(options.getInspector(), + writerOptions.inspector(createEventObjectInspector(findRecId(options.getInspector(), options.getRecordIdColumn()))); + writerOptions.setSchema(createEventSchemaFromTableProperties(options.getTableProperties())); item = new OrcStruct(FIELDS); item.setFieldValue(OPERATION, operation); item.setFieldValue(CURRENT_WRITEID, currentWriteId); http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java index 1a91d4a..e181d8c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java @@ -182,7 +182,7 @@ public class VectorizedOrcAcidRowBatchReader } reporter.setStatus(orcSplit.toString()); - readerOptions = OrcRawRecordMerger.createEventOptions(OrcInputFormat.createOptionsForReader(conf)); + readerOptions = OrcInputFormat.createOptionsForReader(conf); this.offset = orcSplit.getStart(); this.length = orcSplit.getLength(); http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 7c201b6..3e98a09 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.io.AcidOutputFormat; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.BucketCodec; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; @@ -60,6 +61,9 @@ import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; import org.apache.hadoop.hive.ql.txn.compactor.Cleaner; import org.apache.hadoop.hive.ql.txn.compactor.Initiator; import org.apache.hadoop.hive.ql.txn.compactor.Worker; +import org.apache.orc.OrcFile; +import org.apache.orc.Reader; +import org.apache.orc.TypeDescription; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -2168,6 +2172,62 @@ public class TestTxnCommands2 { Assert.assertEquals(stringifyValues(resultData), rs); } + @Test + public void testAcidOrcWritePreservesFieldNames() throws Exception { + // with vectorization + String tableName = "acidorcwritefieldnames"; + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); + runStatementOnDriver("DROP TABLE IF EXISTS " + tableName); + runStatementOnDriver("CREATE TABLE " + tableName + " (a INT, b STRING) CLUSTERED BY (a) INTO " + BUCKET_COUNT + " BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')"); + runStatementOnDriver("INSERT INTO " + tableName + " VALUES (1, 'foo'), (2, 'bar')"); + + tableName = "acidorcwritefieldnames_complex"; + runStatementOnDriver("DROP TABLE IF EXISTS " + tableName); + runStatementOnDriver("CREATE TABLE " + tableName + " (a INT, b STRING, s STRUCT<c:int, si:STRUCT<d:double," + + "e:float>>) CLUSTERED BY (a) INTO " + BUCKET_COUNT + + " BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')"); + runStatementOnDriver("INSERT INTO " + tableName + " select a, b, named_struct('c',10,'si'," + + "named_struct('d',cast(1.0 as double),'e',cast(2.0 as float))) from acidorcwritefieldnames"); + + FileSystem fs = FileSystem.get(hiveConf); + FileStatus[] fileStatuses = fs.globStatus(new Path(TEST_WAREHOUSE_DIR + "/" + tableName + "/" + AcidUtils.DELTA_PREFIX + "*/" + AcidUtils.BUCKET_PREFIX + "*")); + Assert.assertEquals(BUCKET_COUNT, fileStatuses.length); + + OrcFile.ReaderOptions readerOptions = OrcFile.readerOptions(hiveConf); + for (FileStatus fileStatus : fileStatuses) { + Reader r = OrcFile.createReader(fileStatus.getPath(), readerOptions); + TypeDescription rowSchema = r.getSchema().getChildren().get(5); + Assert.assertEquals("struct<a:int,b:string,s:struct<c:int,si:struct<d:double,e:float>>>", rowSchema.toString()); + } + + // without vectorization + tableName = "acidorcwritefieldnames"; + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false); + runStatementOnDriver("DROP TABLE IF EXISTS " + tableName); + runStatementOnDriver("CREATE TABLE " + tableName + " (a INT, b STRING) CLUSTERED BY (a) INTO " + BUCKET_COUNT + " BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')"); + runStatementOnDriver("INSERT INTO " + tableName + " VALUES (1, 'foo'), (2, 'bar')"); + + tableName = "acidorcwritefieldnames_complex"; + runStatementOnDriver("DROP TABLE IF EXISTS " + tableName); + runStatementOnDriver("CREATE TABLE " + tableName + " (a INT, b STRING, s STRUCT<c:int, si:STRUCT<d:double," + + "e:float>>) CLUSTERED BY (a) INTO " + BUCKET_COUNT + + " BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')"); + runStatementOnDriver("INSERT INTO " + tableName + " select a, b, named_struct('c',10,'si'," + + "named_struct('d',cast(1.0 as double),'e',cast(2.0 as float))) from acidorcwritefieldnames"); + + fs = FileSystem.get(hiveConf); + fileStatuses = fs.globStatus(new Path(TEST_WAREHOUSE_DIR + "/" + tableName + "/" + AcidUtils.DELTA_PREFIX + "*/" + AcidUtils.BUCKET_PREFIX + "*")); + Assert.assertEquals(BUCKET_COUNT, fileStatuses.length); + + readerOptions = OrcFile.readerOptions(hiveConf); + for (FileStatus fileStatus : fileStatuses) { + Reader r = OrcFile.createReader(fileStatus.getPath(), readerOptions); + TypeDescription rowSchema = r.getSchema().getChildren().get(5); + Assert.assertEquals("struct<a:int,b:string,s:struct<c:int,si:struct<d:double,e:float>>>", rowSchema.toString()); + } + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); + } + /** * takes raw data and turns it into a string as if from Driver.getResults() * sorts rows in dictionary order http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/acid_nullscan.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/acid_nullscan.q.out b/ql/src/test/results/clientpositive/acid_nullscan.q.out index ccd33ae..c9684dd 100644 --- a/ql/src/test/results/clientpositive/acid_nullscan.q.out +++ b/ql/src/test/results/clientpositive/acid_nullscan.q.out @@ -42,12 +42,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_vectorized_n1 - Statistics: Num rows: 90 Data size: 26090 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 90 Data size: 25960 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 289 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(a) mode: hash @@ -82,7 +82,7 @@ STAGE PLANS: serialization.ddl struct acid_vectorized_n1 { i32 a, string b} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe - totalSize 2609 + totalSize 2596 transactional true transactional_properties default #### A masked pattern was here #### @@ -104,7 +104,7 @@ STAGE PLANS: serialization.ddl struct acid_vectorized_n1 { i32 a, string b} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 2609 + totalSize 2596 transactional true transactional_properties default #### A masked pattern was here #### http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/acid_table_stats.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out index b266794..c028842 100644 --- a/ql/src/test/results/clientpositive/acid_table_stats.q.out +++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out @@ -94,7 +94,7 @@ Table: acid #### A masked pattern was here #### Partition Parameters: numFiles 2 - totalSize 4064 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -132,9 +132,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid - Statistics: Num rows: 83 Data size: 40640 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 40630 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 83 Data size: 40640 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 40630 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -210,7 +210,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 208000 - totalSize 4064 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -261,7 +261,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 208000 - totalSize 4064 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -385,7 +385,7 @@ Table: acid #### A masked pattern was here #### Partition Parameters: numFiles 4 - totalSize 8119 + totalSize 8118 #### A masked pattern was here #### # Storage Information @@ -432,7 +432,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 416000 - totalSize 8119 + totalSize 8118 #### A masked pattern was here #### # Storage Information http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/autoColumnStats_4.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out index d651887..def65ff 100644 --- a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out +++ b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out @@ -200,7 +200,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 2 - totalSize 1907 + totalSize 1899 transactional true transactional_properties default #### A masked pattern was here #### @@ -243,7 +243,7 @@ Table Parameters: COLUMN_STATS_ACCURATE {} bucketing_version 2 numFiles 4 - totalSize 3091 + totalSize 3275 transactional true transactional_properties default #### A masked pattern was here #### http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out index e9fef82..b856b99 100644 --- a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out @@ -45,7 +45,7 @@ STAGE PLANS: alias: acidtbldefault filterExpr: (a = 1) (type: boolean) buckets included: [13,] of 16 - Statistics: Num rows: 1854 Data size: 7052 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1850 Data size: 7036 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false @@ -102,7 +102,7 @@ STAGE PLANS: serialization.ddl struct acidtbldefault { i32 a} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 34863 + totalSize 34795 transactional true transactional_properties default #### A masked pattern was here #### @@ -125,7 +125,7 @@ STAGE PLANS: serialization.ddl struct acidtbldefault { i32 a} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 34863 + totalSize 34795 transactional true transactional_properties default #### A masked pattern was here #### http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out index 66bb2db..be1b4c6 100644 --- a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out @@ -665,11 +665,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over10k_orc_bucketed - Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1237 Data size: 707670 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>) outputColumnNames: ROW__ID - Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1237 Data size: 707670 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>) http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/default_constraint.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/default_constraint.q.out b/ql/src/test/results/clientpositive/llap/default_constraint.q.out index 3365569..775cba8 100644 --- a/ql/src/test/results/clientpositive/llap/default_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/default_constraint.q.out @@ -1498,7 +1498,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 1 - totalSize 1102 + totalSize 1096 transactional true transactional_properties default #### A masked pattern was here #### @@ -1668,7 +1668,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 2 - totalSize 2204 + totalSize 2193 transactional true transactional_properties default #### A masked pattern was here #### @@ -1748,7 +1748,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 2 - totalSize 2204 + totalSize 2193 transactional true transactional_properties default #### A masked pattern was here #### @@ -1925,7 +1925,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3296 + totalSize 3278 transactional true transactional_properties default #### A masked pattern was here #### @@ -2004,7 +2004,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3296 + totalSize 3278 transactional true transactional_properties default #### A masked pattern was here #### @@ -2084,7 +2084,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3296 + totalSize 3278 transactional true transactional_properties default #### A masked pattern was here #### @@ -2660,7 +2660,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 1 - totalSize 1102 + totalSize 1096 transactional true transactional_properties default #### A masked pattern was here #### http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 8dff107..49e5e25 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -94,7 +94,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part - Statistics: Num rows: 160 Data size: 61011 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 160 Data size: 61001 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1906 Basic stats: COMPLETE Column stats: NONE @@ -189,7 +189,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part - Statistics: Num rows: 159 Data size: 104357 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 159 Data size: 104317 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL @@ -380,7 +380,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part_sdpo - Statistics: Num rows: 176 Data size: 67083 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 176 Data size: 67063 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1905 Basic stats: COMPLETE Column stats: NONE @@ -475,7 +475,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part_sdpo - Statistics: Num rows: 171 Data size: 112202 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 171 Data size: 112152 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL @@ -675,7 +675,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_2l_part - Statistics: Num rows: 157 Data size: 60537 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 157 Data size: 60527 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1927 Basic stats: COMPLETE Column stats: NONE @@ -1095,7 +1095,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_2l_part_sdpo - Statistics: Num rows: 157 Data size: 60537 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 157 Data size: 60527 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1927 Basic stats: COMPLETE Column stats: NONE @@ -1515,7 +1515,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_2l_part_sdpo_no_cp - Statistics: Num rows: 97 Data size: 82932 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 97 Data size: 82922 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1860 Basic stats: COMPLETE Column stats: PARTIAL http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out index d61917d..1c0841c 100644 --- a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out @@ -170,7 +170,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 1 - totalSize 295544 + totalSize 295583 transactional true transactional_properties default #### A masked pattern was here #### @@ -205,9 +205,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_ivot - Statistics: Num rows: 5863 Data size: 2955440 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5864 Data size: 2955830 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 5863 Data size: 2955440 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5864 Data size: 2955830 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash @@ -376,7 +376,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 1 - totalSize 1625 + totalSize 1663 transactional true transactional_properties default #### A masked pattern was here #### @@ -411,9 +411,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_ivot - Statistics: Num rows: 32 Data size: 16250 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 16630 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 32 Data size: 16250 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 32 Data size: 16630 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash @@ -509,7 +509,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 2 - totalSize 3250 + totalSize 3326 transactional true transactional_properties default #### A masked pattern was here #### @@ -544,9 +544,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_ivot - Statistics: Num rows: 64 Data size: 32500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 65 Data size: 33260 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 64 Data size: 32500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 65 Data size: 33260 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash @@ -638,7 +638,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 3 - totalSize 298795 + totalSize 298909 transactional true transactional_properties default #### A masked pattern was here #### @@ -673,9 +673,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_ivot - Statistics: Num rows: 5928 Data size: 2987950 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5930 Data size: 2989090 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 5928 Data size: 2987950 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 5930 Data size: 2989090 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out index d7a8a64..237ff00 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out @@ -749,19 +749,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2 - Statistics: Num rows: 43 Data size: 4988 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) @@ -775,7 +775,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int), _col2 (type: decimal(10,2)) mode: hash @@ -1027,19 +1027,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2 - Statistics: Num rows: 43 Data size: 4988 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1053,7 +1053,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int), _col2 (type: decimal(10,2)) mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out index 2f34aae..2b26eba 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out @@ -937,7 +937,7 @@ Table Type: MATERIALIZED_VIEW Table Parameters: bucketing_version 2 numFiles 3 - totalSize 1512 + totalSize 1508 transactional true transactional_properties default #### A masked pattern was here #### @@ -1283,19 +1283,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n2 - Statistics: Num rows: 72 Data size: 8640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 75 Data size: 9000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 24 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)), d (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 24 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 24 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1309,7 +1309,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 40 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 41 Data size: 4920 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col3) keys: _col0 (type: int), _col2 (type: decimal(10,2)) @@ -1532,19 +1532,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n2 - Statistics: Num rows: 88 Data size: 10560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 91 Data size: 10920 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 1080 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)), d (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 1764 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1764 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1641,7 +1641,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 15 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col3) keys: _col0 (type: int), _col2 (type: decimal(10,2)) http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out index 3d5acca..20146f8 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out @@ -400,7 +400,7 @@ Table Type: MATERIALIZED_VIEW Table Parameters: bucketing_version 2 numFiles 2 - totalSize 1076 + totalSize 1071 transactional true transactional_properties default #### A masked pattern was here #### @@ -731,19 +731,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n3 - Statistics: Num rows: 72 Data size: 8352 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 75 Data size: 8700 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 24 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 24 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 24 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) @@ -757,14 +757,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 40 Data size: 4640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 40 Data size: 4640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 40 Data size: 4640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -774,7 +774,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: decimal(10,2)) outputColumnNames: a, c - Statistics: Num rows: 40 Data size: 4640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') mode: hash @@ -943,19 +943,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n3 - Statistics: Num rows: 88 Data size: 10208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 91 Data size: 10556 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 1044 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1160 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) @@ -969,14 +969,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col2 (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -986,7 +986,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: decimal(10,2)) outputColumnNames: a, c - Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out index c85be4a..3b7fb57 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out @@ -749,19 +749,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n0 - Statistics: Num rows: 43 Data size: 4988 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) @@ -775,7 +775,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int), _col2 (type: decimal(10,2)) mode: hash @@ -1027,19 +1027,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cmv_basetable_2_n0 - Statistics: Num rows: 43 Data size: 4988 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1053,7 +1053,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col2 - Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: int), _col2 (type: decimal(10,2)) mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/row__id.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/row__id.q.out b/ql/src/test/results/clientpositive/row__id.q.out index b4f3b8b..bad2d78 100644 --- a/ql/src/test/results/clientpositive/row__id.q.out +++ b/ql/src/test/results/clientpositive/row__id.q.out @@ -62,24 +62,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hello_acid - Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID.writeid (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: bigint) sort order: + - Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -124,17 +124,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hello_acid - Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (ROW__ID.writeid = 3) (type: boolean) - Statistics: Num rows: 39 Data size: 9915 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 39 Data size: 9930 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID.writeid (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 39 Data size: 9915 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 39 Data size: 9930 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 39 Data size: 9915 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 39 Data size: 9930 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out index d10faeb..88499fd 100644 --- a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out +++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out @@ -680,11 +680,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over10k_orc_bucketed_n0 - Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1237 Data size: 707670 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>) outputColumnNames: ROW__ID - Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1237 Data size: 707670 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>) http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out index 7df06ba..9369df8 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out @@ -304,7 +304,7 @@ Stage-3 Output:["_col0","_col1","_col3"] Filter Operator [FIL_9] (rows=10/2 width=316) predicate:((de = 109.23) or (de = 119.23)) - TableScan [TS_0] (rows=86/4 width=316) + TableScan [TS_0] (rows=85/4 width=316) default@acid_uami_n2,acid_uami_n2, ACID table,Tbl:COMPLETE,Col:NONE,Output:["i","de","vc"] PREHOOK: query: select * from acid_uami_n2 order by de