This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 0b4ee968b24 HIVE-28300: Fix AlterTableConcatenate when using Hive-Tez 
(Seonggon Namgung, reviewed by Denys Kuzmenko, Sourabh Badhya)
0b4ee968b24 is described below

commit 0b4ee968b2488796fb0120c4f20a2a4d4ac37799
Author: seonggon <[email protected]>
AuthorDate: Fri Aug 30 20:08:49 2024 +0900

    HIVE-28300: Fix AlterTableConcatenate when using Hive-Tez (Seonggon 
Namgung, reviewed by Denys Kuzmenko, Sourabh Badhya)
    
    Closes #5277
---
 .../test/resources/testconfiguration.properties    |   1 -
 .../AlterTableConcatenateOperation.java            |   4 +-
 .../queries/clientpositive/list_bucket_dml_8.q     |   9 +-
 .../{ => llap}/list_bucket_dml_8.q.out             | 493 ++++++++++-----------
 4 files changed, 239 insertions(+), 268 deletions(-)

diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index cadf4d15006..6768f6b3da5 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -239,7 +239,6 @@ mr.query.files=\
   inputwherefalse.q,\
   join_map_ppr.q,\
   join_vc.q,\
-  list_bucket_dml_8.q,\
   localtimezone.q,\
   mapjoin1.q,\
   mapjoin47.q,\
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
index 88097df79a7..60ff262f225 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
@@ -76,7 +76,9 @@ public class AlterTableConcatenateOperation extends 
DDLOperation<AlterTableConca
 
     Map<Path, List<String>> pathToAliases = new LinkedHashMap<>();
     List<String> inputDirStr = Lists.newArrayList(inputDirList.toString());
-    pathToAliases.put(desc.getInputDir(), inputDirStr);
+    for (Path path: mergeWork.getInputPaths()) {
+      pathToAliases.put(path, inputDirStr);
+    }
     mergeWork.setPathToAliases(pathToAliases);
 
     FileMergeDesc fmd = getFileMergeDesc();
diff --git a/ql/src/test/queries/clientpositive/list_bucket_dml_8.q 
b/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
index 4914b08ecb4..97ef326e861 100644
--- a/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
+++ b/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
@@ -1,11 +1,10 @@
 --! qt:dataset:srcpart
 set hive.mapred.mode=nonstrict;
 set hive.exec.dynamic.partition=true;
-set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+set 
hive.tez.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
 set hive.merge.smallfiles.avgsize=200;
 set mapred.input.dir.recursive=true;
-set hive.merge.mapfiles=false; 
-set hive.merge.mapredfiles=false;
+set hive.merge.tezfiles=false;
 
 -- list bucketing alter table ... concatenate: 
 -- Use list bucketing DML to generate mutilple files in partitions by turning 
off merge
@@ -53,7 +52,7 @@ create table list_bucketing_dynamic_part_n2 (key String, 
value String)
     partitioned by (ds String, hr String) 
     skewed by (key, value) on 
(('484','val_484'),('51','val_14'),('103','val_103'))
     stored as DIRECTORIES
-    STORED AS RCFILE;
+    STORED AS ORC;
 
 -- list bucketing DML without merge. use bucketize to generate a few small 
files.
 explain extended
@@ -73,7 +72,7 @@ alter table list_bucketing_dynamic_part_n2 partition 
(ds='2008-04-08', hr='b1')
 
 desc formatted list_bucketing_dynamic_part_n2 partition (ds='2008-04-08', 
hr='b1');
 
-set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+set hive.tez.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 select count(1) from srcpart where ds = '2008-04-08';
 select count(*) from list_bucketing_dynamic_part_n2;
 explain extended
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_8.q.out 
b/ql/src/test/results/clientpositive/llap/list_bucket_dml_8.q.out
similarity index 57%
rename from ql/src/test/results/clientpositive/list_bucket_dml_8.q.out
rename to ql/src/test/results/clientpositive/llap/list_bucket_dml_8.q.out
index a164b42fceb..18bac2adbf2 100644
--- a/ql/src/test/results/clientpositive/list_bucket_dml_8.q.out
+++ b/ql/src/test/results/clientpositive/llap/list_bucket_dml_8.q.out
@@ -2,7 +2,7 @@ PREHOOK: query: create table list_bucketing_dynamic_part_n2 
(key String, value S
     partitioned by (ds String, hr String) 
     skewed by (key, value) on 
(('484','val_484'),('51','val_14'),('103','val_103'))
     stored as DIRECTORIES
-    STORED AS RCFILE
+    STORED AS ORC
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@list_bucketing_dynamic_part_n2
@@ -10,7 +10,7 @@ POSTHOOK: query: create table list_bucketing_dynamic_part_n2 
(key String, value
     partitioned by (ds String, hr String) 
     skewed by (key, value) on 
(('484','val_484'),('51','val_14'),('103','val_103'))
     stored as DIRECTORIES
-    STORED AS RCFILE
+    STORED AS ORC
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@list_bucketing_dynamic_part_n2
@@ -34,190 +34,203 @@ FROM `default`.`srcpart`
 WHERE `ds` = '2008-04-08'
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
-  Stage-2 depends on stages: Stage-0
+  Stage-2 depends on stages: Stage-1
+  Stage-0 depends on stages: Stage-2
+  Stage-3 depends on stages: Stage-0
 
 STAGE PLANS:
   Stage: Stage-1
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            alias: srcpart
-            filterExpr: (ds = '2008-04-08') (type: boolean)
-            Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE 
Column stats: COMPLETE
-            GatherStats: false
-            Select Operator
-              expressions: key (type: string), value (type: string), 
if(((UDFToDouble(key) % 100.0D) = 0.0D), 'a1', 'b1') (type: string)
-              outputColumnNames: _col0, _col1, _col2
-              Statistics: Num rows: 1000 Data size: 264000 Basic stats: 
COMPLETE Column stats: COMPLETE
-              File Output Operator
-                bucketingVersion: 2
-                compressed: false
-                GlobalTableId: 1
-#### A masked pattern was here ####
-                NumFilesPerFileSink: 1
-                Static Partition Specification: ds=2008-04-08/
-                Statistics: Num rows: 1000 Data size: 264000 Basic stats: 
COMPLETE Column stats: COMPLETE
-#### A masked pattern was here ####
-                table:
-                    input format: 
org.apache.hadoop.hive.ql.io.RCFileInputFormat
-                    output format: 
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: srcpart
+                  filterExpr: (ds = '2008-04-08') (type: boolean)
+                  Statistics: Num rows: 1000 Data size: 178000 Basic stats: 
COMPLETE Column stats: COMPLETE
+                  GatherStats: false
+                  Select Operator
+                    expressions: key (type: string), value (type: string), 
if(((UDFToDouble(key) % 100.0D) = 0.0D), 'a1', 'b1') (type: string)
+                    outputColumnNames: _col0, _col1, _col2
+                    Statistics: Num rows: 1000 Data size: 264000 Basic stats: 
COMPLETE Column stats: COMPLETE
+                    File Output Operator
+                      bucketingVersion: 2
+                      compressed: false
+                      GlobalTableId: 1
+#### A masked pattern was here ####
+                      NumFilesPerFileSink: 1
+                      Static Partition Specification: ds=2008-04-08/
+                      Statistics: Num rows: 1000 Data size: 264000 Basic 
stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+                      table:
+                          input format: 
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                          output format: 
org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                          properties:
+                            bucketing_version 2
+                            column.name.delimiter ,
+                            columns key,value
+                            columns.comments 
+                            columns.types string:string
+#### A masked pattern was here ####
+                            name default.list_bucketing_dynamic_part_n2
+                            partition_columns ds/hr
+                            partition_columns.types string:string
+                            serialization.format 1
+                            serialization.lib 
org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                          name: default.list_bucketing_dynamic_part_n2
+                      TotalFiles: 1
+                      GatherStats: true
+                      MultiFileSpray: false
+                    Select Operator
+                      expressions: _col0 (type: string), _col1 (type: string), 
'2008-04-08' (type: string), _col2 (type: string)
+                      outputColumnNames: key, value, ds, hr
+                      Statistics: Num rows: 1000 Data size: 358000 Basic 
stats: COMPLETE Column stats: COMPLETE
+                      Group By Operator
+                        aggregations: max(length(key)), 
avg(COALESCE(length(key),0)), count(1), count(key), 
compute_bit_vector_hll(key), max(length(value)), 
avg(COALESCE(length(value),0)), count(value), compute_bit_vector_hll(value)
+                        keys: ds (type: string), hr (type: string)
+                        minReductionHashAggr: 0.99
+                        mode: hash
+                        outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6, _col7, _col8, _col9, _col10
+                        Statistics: Num rows: 1 Data size: 652 Basic stats: 
COMPLETE Column stats: COMPLETE
+                        Reduce Output Operator
+                          bucketingVersion: 2
+                          key expressions: _col0 (type: string), _col1 (type: 
string)
+                          null sort order: zz
+                          numBuckets: -1
+                          sort order: ++
+                          Map-reduce partition columns: _col0 (type: string), 
_col1 (type: string)
+                          Statistics: Num rows: 1 Data size: 652 Basic stats: 
COMPLETE Column stats: COMPLETE
+                          tag: -1
+                          value expressions: _col2 (type: int), _col3 (type: 
struct<count:bigint,sum:double,input:int>), _col4 (type: bigint), _col5 (type: 
bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: 
struct<count:bigint,sum:double,input:int>), _col9 (type: bigint), _col10 (type: 
binary)
+                          auto parallelism: true
+            Execution mode: vectorized, llap
+            LLAP IO: all inputs
+            Path -> Alias:
+#### A masked pattern was here ####
+            Path -> Partition:
+#### A masked pattern was here ####
+                Partition
+                  base file name: hr=11
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  partition values:
+                    ds 2008-04-08
+                    hr 11
+                  properties:
+                    column.name.delimiter ,
+                    columns key,value
+                    columns.types string:string
+#### A masked pattern was here ####
+                    name default.srcpart
+                    partition_columns ds/hr
+                    partition_columns.types string:string
+                    serialization.format 1
+                    serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                     properties:
                       bucketing_version 2
                       column.name.delimiter ,
                       columns key,value
-                      columns.comments 
+                      columns.comments 'default','default'
                       columns.types string:string
 #### A masked pattern was here ####
-                      name default.list_bucketing_dynamic_part_n2
+                      name default.srcpart
                       partition_columns ds/hr
                       partition_columns.types string:string
                       serialization.format 1
-                      serialization.lib 
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-                    serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-                    name: default.list_bucketing_dynamic_part_n2
-                TotalFiles: 1
-                GatherStats: true
-                MultiFileSpray: false
-              Select Operator
-                expressions: _col0 (type: string), _col1 (type: string), 
'2008-04-08' (type: string), _col2 (type: string)
-                outputColumnNames: key, value, ds, hr
-                Statistics: Num rows: 1000 Data size: 358000 Basic stats: 
COMPLETE Column stats: COMPLETE
-                Group By Operator
-                  aggregations: max(length(key)), 
avg(COALESCE(length(key),0)), count(1), count(key), 
compute_bit_vector_hll(key), max(length(value)), 
avg(COALESCE(length(value),0)), count(value), compute_bit_vector_hll(value)
-                  keys: ds (type: string), hr (type: string)
-                  minReductionHashAggr: 0.99
-                  mode: hash
-                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8, _col9, _col10
-                  Statistics: Num rows: 1 Data size: 652 Basic stats: COMPLETE 
Column stats: COMPLETE
-                  Reduce Output Operator
-                    bucketingVersion: 2
-                    key expressions: _col0 (type: string), _col1 (type: string)
-                    null sort order: zz
-                    numBuckets: -1
-                    sort order: ++
-                    Map-reduce partition columns: _col0 (type: string), _col1 
(type: string)
-                    Statistics: Num rows: 1 Data size: 652 Basic stats: 
COMPLETE Column stats: COMPLETE
-                    tag: -1
-                    value expressions: _col2 (type: int), _col3 (type: 
struct<count:bigint,sum:double,input:int>), _col4 (type: bigint), _col5 (type: 
bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: 
struct<count:bigint,sum:double,input:int>), _col9 (type: bigint), _col10 (type: 
binary)
-                    auto parallelism: false
-      Execution mode: vectorized
-      Path -> Alias:
-#### A masked pattern was here ####
-      Path -> Partition:
-#### A masked pattern was here ####
-          Partition
-            base file name: hr=11
-            input format: org.apache.hadoop.mapred.TextInputFormat
-            output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-            partition values:
-              ds 2008-04-08
-              hr 11
-            properties:
-              column.name.delimiter ,
-              columns key,value
-              columns.types string:string
-#### A masked pattern was here ####
-              name default.srcpart
-              partition_columns ds/hr
-              partition_columns.types string:string
-              serialization.format 1
-              serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-          
-              input format: org.apache.hadoop.mapred.TextInputFormat
-              output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-              properties:
-                bucketing_version 2
-                column.name.delimiter ,
-                columns key,value
-                columns.comments 'default','default'
-                columns.types string:string
-#### A masked pattern was here ####
-                name default.srcpart
-                partition_columns ds/hr
-                partition_columns.types string:string
-                serialization.format 1
-                serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-              name: default.srcpart
-            name: default.srcpart
-#### A masked pattern was here ####
-          Partition
-            base file name: hr=12
-            input format: org.apache.hadoop.mapred.TextInputFormat
-            output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-            partition values:
-              ds 2008-04-08
-              hr 12
-            properties:
-              column.name.delimiter ,
-              columns key,value
-              columns.types string:string
-#### A masked pattern was here ####
-              name default.srcpart
-              partition_columns ds/hr
-              partition_columns.types string:string
-              serialization.format 1
-              serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-          
-              input format: org.apache.hadoop.mapred.TextInputFormat
-              output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-              properties:
-                bucketing_version 2
-                column.name.delimiter ,
-                columns key,value
-                columns.comments 'default','default'
-                columns.types string:string
-#### A masked pattern was here ####
-                name default.srcpart
-                partition_columns ds/hr
-                partition_columns.types string:string
-                serialization.format 1
-                serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-              name: default.srcpart
-            name: default.srcpart
-      Truncated Path -> Alias:
-        /srcpart/ds=2008-04-08/hr=11 [srcpart]
-        /srcpart/ds=2008-04-08/hr=12 [srcpart]
-      Needs Tagging: false
-      Reduce Operator Tree:
-        Group By Operator
-          aggregations: max(VALUE._col0), avg(VALUE._col1), 
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), 
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7), 
compute_bit_vector_hll(VALUE._col8)
-          keys: KEY._col0 (type: string), KEY._col1 (type: string)
-          mode: mergepartial
-          outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, 
_col7, _col8, _col9, _col10
-          Statistics: Num rows: 1 Data size: 516 Basic stats: COMPLETE Column 
stats: COMPLETE
-          Select Operator
-            expressions: 'STRING' (type: string), UDFToLong(COALESCE(_col2,0)) 
(type: bigint), COALESCE(_col3,0) (type: double), (_col4 - _col5) (type: 
bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: 
binary), 'STRING' (type: string), UDFToLong(COALESCE(_col7,0)) (type: bigint), 
COALESCE(_col8,0) (type: double), (_col4 - _col9) (type: bigint), 
COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: 
binary), _col0 (type: string), _col1 (type [...]
-            outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
-            Statistics: Num rows: 1 Data size: 712 Basic stats: COMPLETE 
Column stats: COMPLETE
-            File Output Operator
-              bucketingVersion: 2
-              compressed: false
-              GlobalTableId: 0
-#### A masked pattern was here ####
-              NumFilesPerFileSink: 1
-              Statistics: Num rows: 1 Data size: 712 Basic stats: COMPLETE 
Column stats: COMPLETE
-#### A masked pattern was here ####
-              table:
-                  input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    name: default.srcpart
+                  name: default.srcpart
+#### A masked pattern was here ####
+                Partition
+                  base file name: hr=12
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  partition values:
+                    ds 2008-04-08
+                    hr 12
                   properties:
-                    bucketing_version -1
-                    columns 
_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13
-                    columns.types 
string:bigint:double:bigint:bigint:binary:string:bigint:double:bigint:bigint:binary:string:string
-                    escape.delim \
-                    hive.serialization.extend.additional.nesting.levels true
-                    serialization.escape.crlf true
+                    column.name.delimiter ,
+                    columns key,value
+                    columns.types string:string
+#### A masked pattern was here ####
+                    name default.srcpart
+                    partition_columns ds/hr
+                    partition_columns.types string:string
                     serialization.format 1
                     serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-              TotalFiles: 1
-              GatherStats: false
-              MultiFileSpray: false
+                
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: 
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    properties:
+                      bucketing_version 2
+                      column.name.delimiter ,
+                      columns key,value
+                      columns.comments 'default','default'
+                      columns.types string:string
+#### A masked pattern was here ####
+                      name default.srcpart
+                      partition_columns ds/hr
+                      partition_columns.types string:string
+                      serialization.format 1
+                      serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    name: default.srcpart
+                  name: default.srcpart
+            Truncated Path -> Alias:
+              /srcpart/ds=2008-04-08/hr=11 [srcpart]
+              /srcpart/ds=2008-04-08/hr=12 [srcpart]
+        Reducer 2 
+            Execution mode: vectorized, llap
+            Needs Tagging: false
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: max(VALUE._col0), avg(VALUE._col1), 
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), 
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7), 
compute_bit_vector_hll(VALUE._col8)
+                keys: KEY._col0 (type: string), KEY._col1 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8, _col9, _col10
+                Statistics: Num rows: 1 Data size: 516 Basic stats: COMPLETE 
Column stats: COMPLETE
+                Select Operator
+                  expressions: 'STRING' (type: string), 
UDFToLong(COALESCE(_col2,0)) (type: bigint), COALESCE(_col3,0) (type: double), 
(_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) 
(type: bigint), _col6 (type: binary), 'STRING' (type: string), 
UDFToLong(COALESCE(_col7,0)) (type: bigint), COALESCE(_col8,0) (type: double), 
(_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) 
(type: bigint), _col10 (type: binary), _col0 (type: string), _col1 [...]
+                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+                  Statistics: Num rows: 1 Data size: 712 Basic stats: COMPLETE 
Column stats: COMPLETE
+                  File Output Operator
+                    bucketingVersion: 2
+                    compressed: false
+                    GlobalTableId: 0
+#### A masked pattern was here ####
+                    NumFilesPerFileSink: 1
+                    Statistics: Num rows: 1 Data size: 712 Basic stats: 
COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+                    table:
+                        input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        properties:
+                          bucketing_version -1
+                          columns 
_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13
+                          columns.types 
string:bigint:double:bigint:bigint:binary:string:bigint:double:bigint:bigint:binary:string:string
+                          escape.delim \
+                          hive.serialization.extend.additional.nesting.levels 
true
+                          serialization.escape.crlf true
+                          serialization.format 1
+                          serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        serde: 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    TotalFiles: 1
+                    GatherStats: false
+                    MultiFileSpray: false
+
+  Stage: Stage-2
+    Dependency Collection
 
   Stage: Stage-0
     Move Operator
@@ -228,8 +241,8 @@ STAGE PLANS:
           replace: true
 #### A masked pattern was here ####
           table:
-              input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-              output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+              input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+              output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               properties:
                 bucketing_version 2
                 column.name.delimiter ,
@@ -241,11 +254,11 @@ STAGE PLANS:
                 partition_columns ds/hr
                 partition_columns.types string:string
                 serialization.format 1
-                serialization.lib 
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-              serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+                serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.list_bucketing_dynamic_part_n2
 
-  Stage: Stage-2
+  Stage: Stage-3
     Stats Work
       Basic Stats Work:
 #### A masked pattern was here ####
@@ -306,14 +319,14 @@ Partition Parameters:
        COLUMN_STATS_ACCURATE   
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
        numFiles                2                   
        numRows                 16                  
-       rawDataSize             136                 
+       rawDataSize             2816                
        totalSize               #Masked#
 #### A masked pattern was here ####
                 
 # Storage Information           
-SerDe Library:         org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe    
 
-InputFormat:           org.apache.hadoop.hive.ql.io.RCFileInputFormat   
-OutputFormat:          org.apache.hadoop.hive.ql.io.RCFileOutputFormat  
+SerDe Library:         org.apache.hadoop.hive.ql.io.orc.OrcSerde        
+InputFormat:           org.apache.hadoop.hive.ql.io.orc.OrcInputFormat  
+OutputFormat:          org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat        
 
 Compressed:            No                       
 Num Buckets:           -1                       
 Bucket Columns:        []                       
@@ -347,14 +360,14 @@ Partition Parameters:
        COLUMN_STATS_ACCURATE   
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
        numFiles                6                   
        numRows                 984                 
-       rawDataSize             9488                
+       rawDataSize             173196              
        totalSize               #Masked#
 #### A masked pattern was here ####
                 
 # Storage Information           
-SerDe Library:         org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe    
 
-InputFormat:           org.apache.hadoop.hive.ql.io.RCFileInputFormat   
-OutputFormat:          org.apache.hadoop.hive.ql.io.RCFileOutputFormat  
+SerDe Library:         org.apache.hadoop.hive.ql.io.orc.OrcSerde        
+InputFormat:           org.apache.hadoop.hive.ql.io.orc.OrcInputFormat  
+OutputFormat:          org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat        
 
 Compressed:            No                       
 Num Buckets:           -1                       
 Bucket Columns:        []                       
@@ -397,14 +410,14 @@ Table:                    list_bucketing_dynamic_part_n2
 Partition Parameters:           
        numFiles                3                   
        numRows                 984                 
-       rawDataSize             9488                
+       rawDataSize             173196              
        totalSize               #Masked#
 #### A masked pattern was here ####
                 
 # Storage Information           
-SerDe Library:         org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe    
 
-InputFormat:           org.apache.hadoop.hive.ql.io.RCFileInputFormat   
-OutputFormat:          org.apache.hadoop.hive.ql.io.RCFileOutputFormat  
+SerDe Library:         org.apache.hadoop.hive.ql.io.orc.OrcSerde        
+InputFormat:           org.apache.hadoop.hive.ql.io.orc.OrcInputFormat  
+OutputFormat:          org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat        
 
 Compressed:            No                       
 Num Buckets:           -1                       
 Bucket Columns:        []                       
@@ -456,59 +469,16 @@ OPTIMIZED SQL: SELECT CAST('484' AS STRING) AS `key`, 
CAST('val_484' AS STRING)
 FROM `default`.`list_bucketing_dynamic_part_n2`
 WHERE `key` = '484' AND `value` = 'val_484'
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
+  Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            alias: list_bucketing_dynamic_part_n2
-            filterExpr: ((key = '484') and (value = 'val_484')) (type: boolean)
-            Statistics: Num rows: 1000 Data size: 546000 Basic stats: COMPLETE 
Column stats: COMPLETE
-            GatherStats: false
-            Filter Operator
-              isSamplingPred: false
-              predicate: ((key = '484') and (value = 'val_484')) (type: 
boolean)
-              Statistics: Num rows: 1 Data size: 546 Basic stats: COMPLETE 
Column stats: COMPLETE
-              Select Operator
-                expressions: '484' (type: string), 'val_484' (type: string), 
ds (type: string), hr (type: string)
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 1 Data size: 546 Basic stats: COMPLETE 
Column stats: COMPLETE
-                File Output Operator
-                  bucketingVersion: 2
-                  compressed: false
-                  GlobalTableId: 0
-#### A masked pattern was here ####
-                  NumFilesPerFileSink: 1
-                  Statistics: Num rows: 1 Data size: 546 Basic stats: COMPLETE 
Column stats: COMPLETE
-#### A masked pattern was here ####
-                  table:
-                      input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-                      output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                      properties:
-                        bucketing_version -1
-                        columns _col0,_col1,_col2,_col3
-                        columns.types string:string:string:string
-                        escape.delim \
-                        hive.serialization.extend.additional.nesting.levels 
true
-                        serialization.escape.crlf true
-                        serialization.format 1
-                        serialization.lib 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                  TotalFiles: 1
-                  GatherStats: false
-                  MultiFileSpray: false
-      Execution mode: vectorized
-      Path -> Alias:
-#### A masked pattern was here ####
-      Path -> Partition:
-#### A masked pattern was here ####
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Partition Description:
           Partition
-            base file name: hr=a1
-            input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-            output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+            input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+            output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
             partition values:
               ds 2008-04-08
               hr a1
@@ -521,11 +491,11 @@ STAGE PLANS:
               partition_columns ds/hr
               partition_columns.types string:string
               serialization.format 1
-              serialization.lib 
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-            serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+              serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+            serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
           
-              input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-              output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+              input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+              output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               properties:
                 bucketing_version 2
                 column.name.delimiter ,
@@ -537,15 +507,13 @@ STAGE PLANS:
                 partition_columns ds/hr
                 partition_columns.types string:string
                 serialization.format 1
-                serialization.lib 
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-              serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+                serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.list_bucketing_dynamic_part_n2
             name: default.list_bucketing_dynamic_part_n2
-#### A masked pattern was here ####
           Partition
-            base file name: hr=b1
-            input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-            output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+            input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+            output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
             partition values:
               ds 2008-04-08
               hr b1
@@ -558,11 +526,11 @@ STAGE PLANS:
               partition_columns ds/hr
               partition_columns.types string:string
               serialization.format 1
-              serialization.lib 
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-            serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+              serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+            serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
           
-              input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
-              output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+              input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+              output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               properties:
                 bucketing_version 2
                 column.name.delimiter ,
@@ -574,19 +542,22 @@ STAGE PLANS:
                 partition_columns ds/hr
                 partition_columns.types string:string
                 serialization.format 1
-                serialization.lib 
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-              serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+                serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.list_bucketing_dynamic_part_n2
             name: default.list_bucketing_dynamic_part_n2
-      Truncated Path -> Alias:
-        /list_bucketing_dynamic_part_n2/ds=2008-04-08/hr=a1 
[list_bucketing_dynamic_part_n2]
-        /list_bucketing_dynamic_part_n2/ds=2008-04-08/hr=b1 
[list_bucketing_dynamic_part_n2]
-
-  Stage: Stage-0
-    Fetch Operator
-      limit: -1
       Processor Tree:
-        ListSink
+        TableScan
+          alias: list_bucketing_dynamic_part_n2
+          filterExpr: ((key = '484') and (value = 'val_484')) (type: boolean)
+          GatherStats: false
+          Filter Operator
+            isSamplingPred: false
+            predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
+            Select Operator
+              expressions: '484' (type: string), 'val_484' (type: string), ds 
(type: string), hr (type: string)
+              outputColumnNames: _col0, _col1, _col2, _col3
+              ListSink
 
 PREHOOK: query: select * from list_bucketing_dynamic_part_n2 where key = '484' 
and value = 'val_484'
 PREHOOK: type: QUERY


Reply via email to