Repository: hive Updated Branches: refs/heads/branch-3 3c71687d1 -> f22412e0d
HIVE-19387: Truncate table for Acid tables conflicts with ResultSet cache (Eugene Koifman, reviewed by Jason Dere) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f22412e0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f22412e0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f22412e0 Branch: refs/heads/branch-3 Commit: f22412e0def2e82fb3c79c03de295dc46ec25d9e Parents: 3c71687 Author: Eugene Koifman <ekoif...@apache.org> Authored: Thu Jul 12 11:08:03 2018 -0700 Committer: Eugene Koifman <ekoif...@apache.org> Committed: Thu Jul 12 11:08:03 2018 -0700 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../hive/ql/parse/DDLSemanticAnalyzer.java | 11 +- .../hadoop/hive/ql/plan/AlterTableDesc.java | 5 + .../hadoop/hive/ql/plan/TruncateTableDesc.java | 33 +- .../clientpositive/results_cache_truncate.q | 61 +++ .../llap/results_cache_truncate.q.out | 470 +++++++++++++++++++ 6 files changed, 579 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 6d8a406..a0037d8 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -625,6 +625,7 @@ minillaplocal.query.files=\ results_cache_quoted_identifiers.q,\ results_cache_temptable.q,\ results_cache_transactional.q,\ + results_cache_truncate.q,\ results_cache_with_masking.q,\ sample10.q,\ sample10_mm.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index f9d6d41..b14648a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1480,7 +1480,13 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { } } - TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName, partSpec, null); + TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName, partSpec, null, table); + if(truncateTblDesc.mayNeedWriteId()) { + if(this.ddlDescWithWriteId != null) { + throw new IllegalStateException("ddlDescWithWriteId is already set: " + this.ddlDescWithWriteId); + } + this.ddlDescWithWriteId = truncateTblDesc; + } DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), truncateTblDesc); Task<? extends Serializable> truncateTask = TaskFactory.get(ddlWork); @@ -1759,6 +1765,9 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), alterTblDesc); if (isPotentialMmSwitch) { + if(this.ddlDescWithWriteId != null) { + throw new IllegalStateException("ddlDescWithWriteId is already set: " + this.ddlDescWithWriteId); + } this.ddlDescWithWriteId = alterTblDesc; ddlWork.setNeedLock(true); // Hmm... why don't many other operations here need locks? } http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java index 0b04c0c..ec04a01 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java @@ -967,4 +967,9 @@ public class AlterTableDesc extends DDLDesc implements Serializable, DDLDesc.DDL public Long getWriteId() { return this.writeId; } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " for " + getFullTableName(); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java index ed68f1a..8c3d852 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java @@ -22,6 +22,9 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -30,25 +33,34 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level; * Truncates managed table or partition */ @Explain(displayName = "Truncate Table or Partition", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class TruncateTableDesc extends DDLDesc { +public class TruncateTableDesc extends DDLDesc implements DDLDesc.DDLDescWithWriteId { private static final long serialVersionUID = 1L; private String tableName; + private String fullTableName; private Map<String, String> partSpec; private List<Integer> columnIndexes; private Path inputDir; private Path outputDir; private ListBucketingCtx lbCtx; private ReplicationSpec replicationSpec; + private long writeId = 0; + private boolean isTransactional; public TruncateTableDesc() { } public TruncateTableDesc(String tableName, Map<String, String> partSpec, ReplicationSpec replicationSpec) { + this(tableName, partSpec, replicationSpec, null); + } + public TruncateTableDesc(String tableName, Map<String, String> partSpec, + ReplicationSpec replicationSpec, Table table) { this.tableName = tableName; this.partSpec = partSpec; this.replicationSpec = replicationSpec; + this.isTransactional = AcidUtils.isTransactionalTable(table); + this.fullTableName = table == null ? tableName : Warehouse.getQualifiedName(table.getTTable()); } @Explain(displayName = "TableName", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) @@ -107,4 +119,23 @@ public class TruncateTableDesc extends DDLDesc { * This can result in a "TRUNCATE IF NEWER THAN" kind of semantic */ public ReplicationSpec getReplicationSpec() { return this.replicationSpec; } + + @Override + public void setWriteId(long writeId) { + this.writeId = writeId; + } + @Override + public String getFullTableName() { + return fullTableName; + } + @Override + public boolean mayNeedWriteId() { + return isTransactional; + } + + @Override + public String toString() { + return this.getClass().getSimpleName() + " for " + getFullTableName(); + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/ql/src/test/queries/clientpositive/results_cache_truncate.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/results_cache_truncate.q b/ql/src/test/queries/clientpositive/results_cache_truncate.q new file mode 100644 index 0000000..f806d49 --- /dev/null +++ b/ql/src/test/queries/clientpositive/results_cache_truncate.q @@ -0,0 +1,61 @@ +--! qt:dataset:src + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create table rct1_1 (key string, value string) stored as orc tblproperties ('transactional'='true'); + +insert into rct1_1 select * from default.src; + +set hive.query.results.cache.enabled=true; +set hive.query.results.cache.nontransactional.tables.enabled=false; + +explain +select count(*) from rct1_1; +select count(*) from rct1_1; + +set test.comment="Query on transactional table should use cache"; +set test.comment; +explain +select count(*) from rct1_1; +select count(*) from rct1_1; + +truncate table rct1_1; + +set test.comment="Table truncated - query cache invalidated"; +set test.comment; +explain +select count(*) from rct1_1; +select count(*) from rct1_1; + +create table rct1_2 (key string, value string) partitioned by (p1 string) stored as orc tblproperties ('transactional'='true'); + +insert into rct1_2 partition (p1='part1') select * from default.src; +insert into rct1_2 partition (p1='part2') select * from default.src; + +explain +select count(*) from rct1_2; +select count(*) from rct1_2; + +set test.comment="Query on transactional table should use cache"; +set test.comment; +explain +select count(*) from rct1_2; +select count(*) from rct1_2; + +truncate table rct1_2 partition (p1='part1'); + +set test.comment="Partition truncated - query cache invalidated"; +set test.comment; +explain +select count(*) from rct1_2; +select count(*) from rct1_2; + +truncate table rct1_2; + +set test.comment="Table truncated - query cache invalidated"; +set test.comment; +explain +select count(*) from rct1_2; +select count(*) from rct1_2; + http://git-wip-us.apache.org/repos/asf/hive/blob/f22412e0/ql/src/test/results/clientpositive/llap/results_cache_truncate.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/results_cache_truncate.q.out b/ql/src/test/results/clientpositive/llap/results_cache_truncate.q.out new file mode 100644 index 0000000..0b7a81a --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/results_cache_truncate.q.out @@ -0,0 +1,470 @@ +PREHOOK: query: create table rct1_1 (key string, value string) stored as orc tblproperties ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@rct1_1 +POSTHOOK: query: create table rct1_1 (key string, value string) stored as orc tblproperties ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@rct1_1 +PREHOOK: query: insert into rct1_1 select * from default.src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@rct1_1 +POSTHOOK: query: insert into rct1_1 select * from default.src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@rct1_1 +POSTHOOK: Lineage: rct1_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: rct1_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +select count(*) from rct1_1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from rct1_1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: rct1_1 + Statistics: Num rows: 500 Data size: 35250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 35250 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from rct1_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@rct1_1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from rct1_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rct1_1 +#### A masked pattern was here #### +500 +test.comment="Query on transactional table should use cache" +PREHOOK: query: explain +select count(*) from rct1_1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from rct1_1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + Cached Query Result: true + +PREHOOK: query: select count(*) from rct1_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@rct1_1 +POSTHOOK: query: select count(*) from rct1_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rct1_1 +500 +PREHOOK: query: truncate table rct1_1 +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@rct1_1 +POSTHOOK: query: truncate table rct1_1 +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@rct1_1 +test.comment="Table truncated - query cache invalidated" +PREHOOK: query: explain +select count(*) from rct1_1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from rct1_1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: rct1_1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from rct1_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@rct1_1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from rct1_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rct1_1 +#### A masked pattern was here #### +0 +PREHOOK: query: create table rct1_2 (key string, value string) partitioned by (p1 string) stored as orc tblproperties ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@rct1_2 +POSTHOOK: query: create table rct1_2 (key string, value string) partitioned by (p1 string) stored as orc tblproperties ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@rct1_2 +PREHOOK: query: insert into rct1_2 partition (p1='part1') select * from default.src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@rct1_2@p1=part1 +POSTHOOK: query: insert into rct1_2 partition (p1='part1') select * from default.src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@rct1_2@p1=part1 +POSTHOOK: Lineage: rct1_2 PARTITION(p1=part1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: rct1_2 PARTITION(p1=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into rct1_2 partition (p1='part2') select * from default.src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@rct1_2@p1=part2 +POSTHOOK: query: insert into rct1_2 partition (p1='part2') select * from default.src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@rct1_2@p1=part2 +POSTHOOK: Lineage: rct1_2 PARTITION(p1=part2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: rct1_2 PARTITION(p1=part2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +select count(*) from rct1_2 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from rct1_2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: rct1_2 + Statistics: Num rows: 1000 Data size: 78450 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 78450 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from rct1_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@rct1_2 +PREHOOK: Input: default@rct1_2@p1=part1 +PREHOOK: Input: default@rct1_2@p1=part2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from rct1_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rct1_2 +POSTHOOK: Input: default@rct1_2@p1=part1 +POSTHOOK: Input: default@rct1_2@p1=part2 +#### A masked pattern was here #### +1000 +test.comment="Query on transactional table should use cache" +PREHOOK: query: explain +select count(*) from rct1_2 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from rct1_2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + Cached Query Result: true + +PREHOOK: query: select count(*) from rct1_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@rct1_2 +PREHOOK: Input: default@rct1_2@p1=part1 +PREHOOK: Input: default@rct1_2@p1=part2 +POSTHOOK: query: select count(*) from rct1_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rct1_2 +POSTHOOK: Input: default@rct1_2@p1=part1 +POSTHOOK: Input: default@rct1_2@p1=part2 +1000 +PREHOOK: query: truncate table rct1_2 partition (p1='part1') +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@rct1_2@p1=part1 +POSTHOOK: query: truncate table rct1_2 partition (p1='part1') +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@rct1_2@p1=part1 +test.comment="Partition truncated - query cache invalidated" +PREHOOK: query: explain +select count(*) from rct1_2 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from rct1_2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: rct1_2 + Statistics: Num rows: 500 Data size: 39200 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 39200 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from rct1_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@rct1_2 +PREHOOK: Input: default@rct1_2@p1=part1 +PREHOOK: Input: default@rct1_2@p1=part2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from rct1_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rct1_2 +POSTHOOK: Input: default@rct1_2@p1=part1 +POSTHOOK: Input: default@rct1_2@p1=part2 +#### A masked pattern was here #### +500 +PREHOOK: query: truncate table rct1_2 +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@rct1_2@p1=part1 +PREHOOK: Output: default@rct1_2@p1=part2 +POSTHOOK: query: truncate table rct1_2 +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@rct1_2@p1=part1 +POSTHOOK: Output: default@rct1_2@p1=part2 +test.comment="Table truncated - query cache invalidated" +PREHOOK: query: explain +select count(*) from rct1_2 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from rct1_2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: rct1_2 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from rct1_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@rct1_2 +PREHOOK: Input: default@rct1_2@p1=part1 +PREHOOK: Input: default@rct1_2@p1=part2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from rct1_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rct1_2 +POSTHOOK: Input: default@rct1_2@p1=part1 +POSTHOOK: Input: default@rct1_2@p1=part2 +#### A masked pattern was here #### +0