HIVE-11340 - Create ORC based table using like clause doesn't copy compression property (Yongzhi Chen, reviewed by Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/57ba795c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/57ba795c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/57ba795c Branch: refs/heads/hbase-metastore Commit: 57ba795cbf98f275b7bae75669d8769aa35d9ee5 Parents: cfe9e48 Author: Yongzhi Chen <yongzhi_c...@hotmail.com> Authored: Tue Aug 11 09:58:30 2015 -0700 Committer: Chao Sun <sunc...@apache.org> Committed: Tue Aug 11 09:58:30 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/io/orc/OrcConf.java | 2 +- .../apache/hadoop/hive/ql/io/orc/OrcSerde.java | 6 +- .../test/queries/clientpositive/create_like.q | 12 ++++ .../results/clientpositive/create_like.q.out | 66 ++++++++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/57ba795c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java index 81b822f..132889c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java @@ -133,7 +133,7 @@ public enum OrcConf { private String lookupValue(Properties tbl, Configuration conf) { String result = null; if (tbl != null) { - result = conf.get(attribute); + result = tbl.getProperty(attribute); } if (result == null && conf != null) { result = conf.get(attribute); http://git-wip-us.apache.org/repos/asf/hive/blob/57ba795c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java index a381443..8beff4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java @@ -22,6 +22,7 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Properties; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -42,7 +43,7 @@ import org.apache.hadoop.io.Writable; * A serde class for ORC. * It transparently passes the object to/from the ORC file reader/writer. */ -@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES}) +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, OrcSerde.COMPRESSION}) public class OrcSerde implements SerDe, VectorizedSerde { private static final Log LOG = LogFactory.getLog(OrcSerde.class); @@ -51,6 +52,7 @@ public class OrcSerde implements SerDe, VectorizedSerde { private ObjectInspector inspector = null; private VectorizedOrcSerde vos = null; + public static final String COMPRESSION = "orc.compress"; final class OrcSerdeRow implements Writable { Object realRow; @@ -82,6 +84,8 @@ public class OrcSerde implements SerDe, VectorizedSerde { // NOTE: if "columns.types" is missing, all columns will be of String type String columnTypeProperty = table.getProperty(serdeConstants.LIST_COLUMN_TYPES); + String compressType = OrcConf.COMPRESS.getString(table, conf); + // Parse the configuration parameters ArrayList<String> columnNames = new ArrayList<String>(); if (columnNameProperty != null && columnNameProperty.length() > 0) { http://git-wip-us.apache.org/repos/asf/hive/blob/57ba795c/ql/src/test/queries/clientpositive/create_like.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/create_like.q b/ql/src/test/queries/clientpositive/create_like.q index 3b04702..bd39731 100644 --- a/ql/src/test/queries/clientpositive/create_like.q +++ b/ql/src/test/queries/clientpositive/create_like.q @@ -83,3 +83,15 @@ DESCRIBE FORMATTED table6; drop table table5; +create table orc_table ( +time string) +stored as ORC tblproperties ("orc.compress"="SNAPPY"); + +create table orc_table_using_like like orc_table; + +describe formatted orc_table_using_like; + +drop table orc_table_using_like; + +drop table orc_table; + http://git-wip-us.apache.org/repos/asf/hive/blob/57ba795c/ql/src/test/results/clientpositive/create_like.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/create_like.q.out b/ql/src/test/results/clientpositive/create_like.q.out index c93b134..a373178 100644 --- a/ql/src/test/results/clientpositive/create_like.q.out +++ b/ql/src/test/results/clientpositive/create_like.q.out @@ -579,3 +579,69 @@ POSTHOOK: query: drop table table5 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@table5 POSTHOOK: Output: default@table5 +PREHOOK: query: create table orc_table ( +time string) +stored as ORC tblproperties ("orc.compress"="SNAPPY") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_table +POSTHOOK: query: create table orc_table ( +time string) +stored as ORC tblproperties ("orc.compress"="SNAPPY") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_table +PREHOOK: query: create table orc_table_using_like like orc_table +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_table_using_like +POSTHOOK: query: create table orc_table_using_like like orc_table +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_table_using_like +PREHOOK: query: describe formatted orc_table_using_like +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orc_table_using_like +POSTHOOK: query: describe formatted orc_table_using_like +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orc_table_using_like +# col_name data_type comment + +time string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + orc.compress SNAPPY +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table orc_table_using_like +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orc_table_using_like +PREHOOK: Output: default@orc_table_using_like +POSTHOOK: query: drop table orc_table_using_like +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orc_table_using_like +POSTHOOK: Output: default@orc_table_using_like +PREHOOK: query: drop table orc_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orc_table +PREHOOK: Output: default@orc_table +POSTHOOK: query: drop table orc_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orc_table +POSTHOOK: Output: default@orc_table