HIVE-11239: Test encryption_insert_partition_static.q fails with different output results on other environments(Sergio Pena, reviewed by Ferdinand Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a65bcbdf Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a65bcbdf Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a65bcbdf Branch: refs/heads/spark Commit: a65bcbdf463903a5a9650693d597a4b711abea2f Parents: 21aecbc Author: Ferdinand Xu <cheng.a...@intel.com> Authored: Tue Jul 14 04:50:15 2015 -0400 Committer: Ferdinand Xu <cheng.a...@intel.com> Committed: Tue Jul 14 04:50:15 2015 -0400 ---------------------------------------------------------------------- .../encryption_insert_partition_static.q | 17 - .../encryption_insert_partition_static.q.out | 739 +------------------ 2 files changed, 11 insertions(+), 745 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a65bcbdf/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q b/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q index c5769a6..69687df 100644 --- a/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q +++ b/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q @@ -18,11 +18,6 @@ create table unencryptedTable(key string, value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); -- insert encrypted table from values -explain extended insert into table encryptedTable partition - (ds='today') values - ('501', 'val_501'), - ('502', 'val_502'); - insert into table encryptedTable partition (ds='today') values ('501', 'val_501'), @@ -31,27 +26,15 @@ insert into table encryptedTable partition select * from encryptedTable order by key; -- insert encrypted table from unencrypted source -explain extended -insert into table encryptedTable partition (ds='yesterday') -select * from src where key in ('238', '86'); - insert into table encryptedTable partition (ds='yesterday') select * from src where key in ('238', '86'); select * from encryptedTable order by key; -- insert unencrypted table from encrypted source -explain extended insert into table unencryptedTable partition (ds='today') select key, value from encryptedTable where ds='today'; -insert into table unencryptedTable partition (ds='today') -select key, value from encryptedTable where ds='today'; - -explain extended -insert into table unencryptedTable partition (ds='yesterday') -select key, value from encryptedTable where ds='yesterday'; - insert into table unencryptedTable partition (ds='yesterday') select key, value from encryptedTable where ds='yesterday'; http://git-wip-us.apache.org/repos/asf/hive/blob/a65bcbdf/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out index b10610c..c2f0ddc 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out @@ -39,187 +39,23 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@unencryptedTable PREHOOK: query: -- insert encrypted table from values -explain extended insert into table encryptedTable partition +insert into table encryptedTable partition (ds='today') values ('501', 'val_501'), ('502', 'val_502') PREHOOK: type: QUERY -POSTHOOK: query: -- insert encrypted table from values -explain extended insert into table encryptedTable partition - (ds='today') values - ('501', 'val_501'), - ('502', 'val_502') -POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - -TOK_QUERY - TOK_FROM - null - null - Values__Tmp__Table__1 - TOK_INSERT - TOK_INSERT_INTO - TOK_TAB - TOK_TABNAME - encryptedTable - TOK_PARTSPEC - TOK_PARTVAL - ds - 'today' - TOK_SELECT - TOK_SELEXPR - TOK_ALLCOLREF - - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: values__tmp__table__1 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Select Operator - expressions: tmp_values_col1 (type: string), tmp_values_col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) - auto parallelism: false - Path -> Alias: -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging - Path -> Partition: -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging - Partition - base file name: Values__Tmp__Table__1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns tmp_values_col1,tmp_values_col2 - columns.comments - columns.types string:string -#### A masked pattern was here #### -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging - name default.values__tmp__table__1 - serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns tmp_values_col1,tmp_values_col2 - columns.comments - columns.types string:string -#### A masked pattern was here #### -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging - name default.values__tmp__table__1 - serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.values__tmp__table__1 - name: default.values__tmp__table__1 - Truncated Path -> Alias: -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=today/.hive-staging - NumFilesPerFileSink: 1 - Static Partition Specification: ds=today/ - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=today/.hive-staging - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.encryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct encryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.encryptedtable - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - - Stage: Stage-0 - Move Operator - tables: - partition: - ds today - replace: false -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=today/.hive-staging - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.encryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct encryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.encryptedtable - - Stage: Stage-2 - Stats-Aggr Operator -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=today/.hive-staging - -PREHOOK: query: insert into table encryptedTable partition - (ds='today') values - ('501', 'val_501'), - ('502', 'val_502') -PREHOOK: type: QUERY -PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Input: default@values__tmp__table__1 PREHOOK: Output: default@encryptedtable@ds=today -POSTHOOK: query: insert into table encryptedTable partition +POSTHOOK: query: -- insert encrypted table from values +insert into table encryptedTable partition (ds='today') values ('501', 'val_501'), ('502', 'val_502') POSTHOOK: type: QUERY -POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Input: default@values__tmp__table__1 POSTHOOK: Output: default@encryptedtable@ds=today -POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).key SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).key SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] PREHOOK: query: select * from encryptedTable order by key PREHOOK: type: QUERY PREHOOK: Input: default@encryptedtable @@ -233,196 +69,13 @@ POSTHOOK: Input: default@encryptedtable@ds=today 501 val_501 today 502 val_502 today PREHOOK: query: -- insert encrypted table from unencrypted source -explain extended insert into table encryptedTable partition (ds='yesterday') select * from src where key in ('238', '86') PREHOOK: type: QUERY -POSTHOOK: query: -- insert encrypted table from unencrypted source -explain extended -insert into table encryptedTable partition (ds='yesterday') -select * from src where key in ('238', '86') -POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - -TOK_QUERY - TOK_FROM - TOK_TABREF - TOK_TABNAME - src - TOK_INSERT - TOK_INSERT_INTO - TOK_TAB - TOK_TABNAME - encryptedTable - TOK_PARTSPEC - TOK_PARTVAL - ds - 'yesterday' - TOK_SELECT - TOK_SELEXPR - TOK_ALLCOLREF - TOK_WHERE - TOK_FUNCTION - in - TOK_TABLE_OR_COL - key - '238' - '86' - - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key) IN ('238', '86') (type: boolean) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 0 - rawDataSize 0 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 0 - rawDataSize 0 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [src] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=yesterday/.hive-staging - NumFilesPerFileSink: 1 - Static Partition Specification: ds=yesterday/ - Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=yesterday/.hive-staging - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.encryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct encryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.encryptedtable - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - - Stage: Stage-0 - Move Operator - tables: - partition: - ds yesterday - replace: false -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=yesterday/.hive-staging - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.encryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct encryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.encryptedtable - - Stage: Stage-2 - Stats-Aggr Operator -#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/ds=yesterday/.hive-staging - -PREHOOK: query: insert into table encryptedTable partition (ds='yesterday') -select * from src where key in ('238', '86') -PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@encryptedtable@ds=yesterday -POSTHOOK: query: insert into table encryptedTable partition (ds='yesterday') +POSTHOOK: query: -- insert encrypted table from unencrypted source +insert into table encryptedTable partition (ds='yesterday') select * from src where key in ('238', '86') POSTHOOK: type: QUERY POSTHOOK: Input: default@src @@ -447,199 +100,14 @@ POSTHOOK: Input: default@encryptedtable@ds=yesterday 502 val_502 today 86 val_86 yesterday PREHOOK: query: -- insert unencrypted table from encrypted source -explain extended insert into table unencryptedTable partition (ds='today') select key, value from encryptedTable where ds='today' PREHOOK: type: QUERY -POSTHOOK: query: -- insert unencrypted table from encrypted source -explain extended -insert into table unencryptedTable partition (ds='today') -select key, value from encryptedTable where ds='today' -POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - -TOK_QUERY - TOK_FROM - TOK_TABREF - TOK_TABNAME - encryptedTable - TOK_INSERT - TOK_INSERT_INTO - TOK_TAB - TOK_TABNAME - unencryptedTable - TOK_PARTSPEC - TOK_PARTVAL - ds - 'today' - TOK_SELECT - TOK_SELEXPR - TOK_TABLE_OR_COL - key - TOK_SELEXPR - TOK_TABLE_OR_COL - value - TOK_WHERE - = - TOK_TABLE_OR_COL - ds - 'today' - - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: encryptedtable - Statistics: Num rows: 6 Data size: 1351 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 1351 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 1351 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: ds=today - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - partition values: - ds today - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.encryptedtable - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct encryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 1351 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.encryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct encryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.encryptedtable - name: default.encryptedtable - Truncated Path -> Alias: - /encryptedTable/ds=today [encryptedtable] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 1351 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging - NumFilesPerFileSink: 1 - Static Partition Specification: ds=today/ - Statistics: Num rows: 6 Data size: 1351 Basic stats: COMPLETE Column stats: NONE -#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.unencryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct unencryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.unencryptedtable - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - - Stage: Stage-0 - Move Operator - tables: - partition: - ds today - replace: false -#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.unencryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct unencryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.unencryptedtable - - Stage: Stage-2 - Stats-Aggr Operator -#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging - -PREHOOK: query: insert into table unencryptedTable partition (ds='today') -select key, value from encryptedTable where ds='today' -PREHOOK: type: QUERY PREHOOK: Input: default@encryptedtable PREHOOK: Input: default@encryptedtable@ds=today PREHOOK: Output: default@unencryptedtable@ds=today -POSTHOOK: query: insert into table unencryptedTable partition (ds='today') +POSTHOOK: query: -- insert unencrypted table from encrypted source +insert into table unencryptedTable partition (ds='today') select key, value from encryptedTable where ds='today' POSTHOOK: type: QUERY POSTHOOK: Input: default@encryptedtable @@ -647,191 +115,6 @@ POSTHOOK: Input: default@encryptedtable@ds=today POSTHOOK: Output: default@unencryptedtable@ds=today POSTHOOK: Lineage: unencryptedtable PARTITION(ds=today).key SIMPLE [(encryptedtable)encryptedtable.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: unencryptedtable PARTITION(ds=today).value SIMPLE [(encryptedtable)encryptedtable.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: explain extended -insert into table unencryptedTable partition (ds='yesterday') -select key, value from encryptedTable where ds='yesterday' -PREHOOK: type: QUERY -POSTHOOK: query: explain extended -insert into table unencryptedTable partition (ds='yesterday') -select key, value from encryptedTable where ds='yesterday' -POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - -TOK_QUERY - TOK_FROM - TOK_TABREF - TOK_TABNAME - encryptedTable - TOK_INSERT - TOK_INSERT_INTO - TOK_TAB - TOK_TABNAME - unencryptedTable - TOK_PARTSPEC - TOK_PARTVAL - ds - 'yesterday' - TOK_SELECT - TOK_SELEXPR - TOK_TABLE_OR_COL - key - TOK_SELEXPR - TOK_TABLE_OR_COL - value - TOK_WHERE - = - TOK_TABLE_OR_COL - ds - 'yesterday' - - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: encryptedtable - Statistics: Num rows: 6 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: ds=yesterday - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - partition values: - ds yesterday - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.encryptedtable - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct encryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 1372 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.encryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct encryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.encryptedtable - name: default.encryptedtable - Truncated Path -> Alias: - /encryptedTable/ds=yesterday [encryptedtable] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=yesterday/.hive-staging - NumFilesPerFileSink: 1 - Static Partition Specification: ds=yesterday/ - Statistics: Num rows: 6 Data size: 1372 Basic stats: COMPLETE Column stats: NONE -#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=yesterday/.hive-staging - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.unencryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct unencryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.unencryptedtable - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - - Stage: Stage-0 - Move Operator - tables: - partition: - ds yesterday - replace: false -#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=yesterday/.hive-staging - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.unencryptedtable - partition_columns ds - partition_columns.types string - serialization.ddl struct unencryptedtable { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - transactional true -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.unencryptedtable - - Stage: Stage-2 - Stats-Aggr Operator -#### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=yesterday/.hive-staging - PREHOOK: query: insert into table unencryptedTable partition (ds='yesterday') select key, value from encryptedTable where ds='yesterday' PREHOOK: type: QUERY