This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0b4ee968b24 HIVE-28300: Fix AlterTableConcatenate when using Hive-Tez
(Seonggon Namgung, reviewed by Denys Kuzmenko, Sourabh Badhya)
0b4ee968b24 is described below
commit 0b4ee968b2488796fb0120c4f20a2a4d4ac37799
Author: seonggon <[email protected]>
AuthorDate: Fri Aug 30 20:08:49 2024 +0900
HIVE-28300: Fix AlterTableConcatenate when using Hive-Tez (Seonggon
Namgung, reviewed by Denys Kuzmenko, Sourabh Badhya)
Closes #5277
---
.../test/resources/testconfiguration.properties | 1 -
.../AlterTableConcatenateOperation.java | 4 +-
.../queries/clientpositive/list_bucket_dml_8.q | 9 +-
.../{ => llap}/list_bucket_dml_8.q.out | 493 ++++++++++-----------
4 files changed, 239 insertions(+), 268 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index cadf4d15006..6768f6b3da5 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -239,7 +239,6 @@ mr.query.files=\
inputwherefalse.q,\
join_map_ppr.q,\
join_vc.q,\
- list_bucket_dml_8.q,\
localtimezone.q,\
mapjoin1.q,\
mapjoin47.q,\
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
index 88097df79a7..60ff262f225 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
@@ -76,7 +76,9 @@ public class AlterTableConcatenateOperation extends
DDLOperation<AlterTableConca
Map<Path, List<String>> pathToAliases = new LinkedHashMap<>();
List<String> inputDirStr = Lists.newArrayList(inputDirList.toString());
- pathToAliases.put(desc.getInputDir(), inputDirStr);
+ for (Path path: mergeWork.getInputPaths()) {
+ pathToAliases.put(path, inputDirStr);
+ }
mergeWork.setPathToAliases(pathToAliases);
FileMergeDesc fmd = getFileMergeDesc();
diff --git a/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
b/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
index 4914b08ecb4..97ef326e861 100644
--- a/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
+++ b/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
@@ -1,11 +1,10 @@
--! qt:dataset:srcpart
set hive.mapred.mode=nonstrict;
set hive.exec.dynamic.partition=true;
-set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+set
hive.tez.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
set hive.merge.smallfiles.avgsize=200;
set mapred.input.dir.recursive=true;
-set hive.merge.mapfiles=false;
-set hive.merge.mapredfiles=false;
+set hive.merge.tezfiles=false;
-- list bucketing alter table ... concatenate:
-- Use list bucketing DML to generate mutilple files in partitions by turning
off merge
@@ -53,7 +52,7 @@ create table list_bucketing_dynamic_part_n2 (key String,
value String)
partitioned by (ds String, hr String)
skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
stored as DIRECTORIES
- STORED AS RCFILE;
+ STORED AS ORC;
-- list bucketing DML without merge. use bucketize to generate a few small
files.
explain extended
@@ -73,7 +72,7 @@ alter table list_bucketing_dynamic_part_n2 partition
(ds='2008-04-08', hr='b1')
desc formatted list_bucketing_dynamic_part_n2 partition (ds='2008-04-08',
hr='b1');
-set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+set hive.tez.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
select count(1) from srcpart where ds = '2008-04-08';
select count(*) from list_bucketing_dynamic_part_n2;
explain extended
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_8.q.out
b/ql/src/test/results/clientpositive/llap/list_bucket_dml_8.q.out
similarity index 57%
rename from ql/src/test/results/clientpositive/list_bucket_dml_8.q.out
rename to ql/src/test/results/clientpositive/llap/list_bucket_dml_8.q.out
index a164b42fceb..18bac2adbf2 100644
--- a/ql/src/test/results/clientpositive/list_bucket_dml_8.q.out
+++ b/ql/src/test/results/clientpositive/llap/list_bucket_dml_8.q.out
@@ -2,7 +2,7 @@ PREHOOK: query: create table list_bucketing_dynamic_part_n2
(key String, value S
partitioned by (ds String, hr String)
skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
stored as DIRECTORIES
- STORED AS RCFILE
+ STORED AS ORC
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@list_bucketing_dynamic_part_n2
@@ -10,7 +10,7 @@ POSTHOOK: query: create table list_bucketing_dynamic_part_n2
(key String, value
partitioned by (ds String, hr String)
skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
stored as DIRECTORIES
- STORED AS RCFILE
+ STORED AS ORC
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@list_bucketing_dynamic_part_n2
@@ -34,190 +34,203 @@ FROM `default`.`srcpart`
WHERE `ds` = '2008-04-08'
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: srcpart
- filterExpr: (ds = '2008-04-08') (type: boolean)
- Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE
Column stats: COMPLETE
- GatherStats: false
- Select Operator
- expressions: key (type: string), value (type: string),
if(((UDFToDouble(key) % 100.0D) = 0.0D), 'a1', 'b1') (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1000 Data size: 264000 Basic stats:
COMPLETE Column stats: COMPLETE
- File Output Operator
- bucketingVersion: 2
- compressed: false
- GlobalTableId: 1
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
- Static Partition Specification: ds=2008-04-08/
- Statistics: Num rows: 1000 Data size: 264000 Basic stats:
COMPLETE Column stats: COMPLETE
-#### A masked pattern was here ####
- table:
- input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart
+ filterExpr: (ds = '2008-04-08') (type: boolean)
+ Statistics: Num rows: 1000 Data size: 178000 Basic stats:
COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string),
if(((UDFToDouble(key) % 100.0D) = 0.0D), 'a1', 'b1') (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1000 Data size: 264000 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Static Partition Specification: ds=2008-04-08/
+ Statistics: Num rows: 1000 Data size: 264000 Basic
stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_dynamic_part_n2
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.list_bucketing_dynamic_part_n2
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string),
'2008-04-08' (type: string), _col2 (type: string)
+ outputColumnNames: key, value, ds, hr
+ Statistics: Num rows: 1000 Data size: 358000 Basic
stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: max(length(key)),
avg(COALESCE(length(key),0)), count(1), count(key),
compute_bit_vector_hll(key), max(length(value)),
avg(COALESCE(length(value),0)), count(value), compute_bit_vector_hll(value)
+ keys: ds (type: string), hr (type: string)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7, _col8, _col9, _col10
+ Statistics: Num rows: 1 Data size: 652 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: string), _col1 (type:
string)
+ null sort order: zz
+ numBuckets: -1
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string),
_col1 (type: string)
+ Statistics: Num rows: 1 Data size: 652 Basic stats:
COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col2 (type: int), _col3 (type:
struct<count:bigint,sum:double,input:int>), _col4 (type: bigint), _col5 (type:
bigint), _col6 (type: binary), _col7 (type: int), _col8 (type:
struct<count:bigint,sum:double,input:int>), _col9 (type: bigint), _col10 (type:
binary)
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: hr=11
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ column.name.delimiter ,
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version 2
column.name.delimiter ,
columns key,value
- columns.comments
+ columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.list_bucketing_dynamic_part_n2
+ name default.srcpart
partition_columns ds/hr
partition_columns.types string:string
serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.list_bucketing_dynamic_part_n2
- TotalFiles: 1
- GatherStats: true
- MultiFileSpray: false
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string),
'2008-04-08' (type: string), _col2 (type: string)
- outputColumnNames: key, value, ds, hr
- Statistics: Num rows: 1000 Data size: 358000 Basic stats:
COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: max(length(key)),
avg(COALESCE(length(key),0)), count(1), count(key),
compute_bit_vector_hll(key), max(length(value)),
avg(COALESCE(length(value),0)), count(value), compute_bit_vector_hll(value)
- keys: ds (type: string), hr (type: string)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 652 Basic stats: COMPLETE
Column stats: COMPLETE
- Reduce Output Operator
- bucketingVersion: 2
- key expressions: _col0 (type: string), _col1 (type: string)
- null sort order: zz
- numBuckets: -1
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1
(type: string)
- Statistics: Num rows: 1 Data size: 652 Basic stats:
COMPLETE Column stats: COMPLETE
- tag: -1
- value expressions: _col2 (type: int), _col3 (type:
struct<count:bigint,sum:double,input:int>), _col4 (type: bigint), _col5 (type:
bigint), _col6 (type: binary), _col7 (type: int), _col8 (type:
struct<count:bigint,sum:double,input:int>), _col9 (type: bigint), _col10 (type:
binary)
- auto parallelism: false
- Execution mode: vectorized
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: hr=11
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- hr 11
- properties:
- column.name.delimiter ,
- columns key,value
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucketing_version 2
- column.name.delimiter ,
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
-#### A masked pattern was here ####
- Partition
- base file name: hr=12
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- hr 12
- properties:
- column.name.delimiter ,
- columns key,value
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucketing_version 2
- column.name.delimiter ,
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
- Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [srcpart]
- /srcpart/ds=2008-04-08/hr=12 [srcpart]
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: max(VALUE._col0), avg(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8)
- keys: KEY._col0 (type: string), KEY._col1 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6,
_col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 516 Basic stats: COMPLETE Column
stats: COMPLETE
- Select Operator
- expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0))
(type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type:
bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type:
binary), 'STRING' (type: string), UDFToLong(COALESCE(_col7,0)) (type: bigint),
COALESCE(_col8,0) (type: double), (_col4 - _col9) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type:
binary), _col0 (type: string), _col1 (type [...]
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 1 Data size: 712 Basic stats: COMPLETE
Column stats: COMPLETE
- File Output Operator
- bucketingVersion: 2
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 712 Basic stats: COMPLETE
Column stats: COMPLETE
-#### A masked pattern was here ####
- table:
- input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.srcpart
+ name: default.srcpart
+#### A masked pattern was here ####
+ Partition
+ base file name: hr=12
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 12
properties:
- bucketing_version -1
- columns
_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13
- columns.types
string:bigint:double:bigint:bigint:binary:string:bigint:double:bigint:bigint:binary:string:string
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.escape.crlf true
+ column.name.delimiter ,
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
serialization.format 1
serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.srcpart
+ name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [srcpart]
+ /srcpart/ds=2008-04-08/hr=12 [srcpart]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), avg(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10
+ Statistics: Num rows: 1 Data size: 516 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'STRING' (type: string),
UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double),
(_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0)
(type: bigint), _col6 (type: binary), 'STRING' (type: string),
UDFToLong(COALESCE(_col7,0)) (type: bigint), COALESCE(_col8,0) (type: double),
(_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0)
(type: bigint), _col10 (type: binary), _col0 (type: string), _col1 [...]
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Statistics: Num rows: 1 Data size: 712 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 712 Basic stats:
COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns
_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13
+ columns.types
string:bigint:double:bigint:bigint:binary:string:bigint:double:bigint:bigint:binary:string:string
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels
true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-2
+ Dependency Collection
Stage: Stage-0
Move Operator
@@ -228,8 +241,8 @@ STAGE PLANS:
replace: true
#### A masked pattern was here ####
table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
properties:
bucketing_version 2
column.name.delimiter ,
@@ -241,11 +254,11 @@ STAGE PLANS:
partition_columns ds/hr
partition_columns.types string:string
serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.list_bucketing_dynamic_part_n2
- Stage: Stage-2
+ Stage: Stage-3
Stats Work
Basic Stats Work:
#### A masked pattern was here ####
@@ -306,14 +319,14 @@ Partition Parameters:
COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 2
numRows 16
- rawDataSize 136
+ rawDataSize 2816
totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
-SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
Compressed: No
Num Buckets: -1
Bucket Columns: []
@@ -347,14 +360,14 @@ Partition Parameters:
COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 6
numRows 984
- rawDataSize 9488
+ rawDataSize 173196
totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
-SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
Compressed: No
Num Buckets: -1
Bucket Columns: []
@@ -397,14 +410,14 @@ Table: list_bucketing_dynamic_part_n2
Partition Parameters:
numFiles 3
numRows 984
- rawDataSize 9488
+ rawDataSize 173196
totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
-SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
Compressed: No
Num Buckets: -1
Bucket Columns: []
@@ -456,59 +469,16 @@ OPTIMIZED SQL: SELECT CAST('484' AS STRING) AS `key`,
CAST('val_484' AS STRING)
FROM `default`.`list_bucketing_dynamic_part_n2`
WHERE `key` = '484' AND `value` = 'val_484'
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: list_bucketing_dynamic_part_n2
- filterExpr: ((key = '484') and (value = 'val_484')) (type: boolean)
- Statistics: Num rows: 1000 Data size: 546000 Basic stats: COMPLETE
Column stats: COMPLETE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: ((key = '484') and (value = 'val_484')) (type:
boolean)
- Statistics: Num rows: 1 Data size: 546 Basic stats: COMPLETE
Column stats: COMPLETE
- Select Operator
- expressions: '484' (type: string), 'val_484' (type: string),
ds (type: string), hr (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 546 Basic stats: COMPLETE
Column stats: COMPLETE
- File Output Operator
- bucketingVersion: 2
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 546 Basic stats: COMPLETE
Column stats: COMPLETE
-#### A masked pattern was here ####
- table:
- input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- properties:
- bucketing_version -1
- columns _col0,_col1,_col2,_col3
- columns.types string:string:string:string
- escape.delim \
- hive.serialization.extend.additional.nesting.levels
true
- serialization.escape.crlf true
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
- Execution mode: vectorized
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Partition Description:
Partition
- base file name: hr=a1
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
partition values:
ds 2008-04-08
hr a1
@@ -521,11 +491,11 @@ STAGE PLANS:
partition_columns ds/hr
partition_columns.types string:string
serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
properties:
bucketing_version 2
column.name.delimiter ,
@@ -537,15 +507,13 @@ STAGE PLANS:
partition_columns ds/hr
partition_columns.types string:string
serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.list_bucketing_dynamic_part_n2
name: default.list_bucketing_dynamic_part_n2
-#### A masked pattern was here ####
Partition
- base file name: hr=b1
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
partition values:
ds 2008-04-08
hr b1
@@ -558,11 +526,11 @@ STAGE PLANS:
partition_columns ds/hr
partition_columns.types string:string
serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
properties:
bucketing_version 2
column.name.delimiter ,
@@ -574,19 +542,22 @@ STAGE PLANS:
partition_columns ds/hr
partition_columns.types string:string
serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.list_bucketing_dynamic_part_n2
name: default.list_bucketing_dynamic_part_n2
- Truncated Path -> Alias:
- /list_bucketing_dynamic_part_n2/ds=2008-04-08/hr=a1
[list_bucketing_dynamic_part_n2]
- /list_bucketing_dynamic_part_n2/ds=2008-04-08/hr=b1
[list_bucketing_dynamic_part_n2]
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
Processor Tree:
- ListSink
+ TableScan
+ alias: list_bucketing_dynamic_part_n2
+ filterExpr: ((key = '484') and (value = 'val_484')) (type: boolean)
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
+ Select Operator
+ expressions: '484' (type: string), 'val_484' (type: string), ds
(type: string), hr (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ ListSink
PREHOOK: query: select * from list_bucketing_dynamic_part_n2 where key = '484'
and value = 'val_484'
PREHOOK: type: QUERY