Repository: hive Updated Branches: refs/heads/master c29038af9 -> 2ff9c5229
HIVE-20700 : Add config to diable rollup for druid tables (Nishant Bangarwa via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2ff9c522 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2ff9c522 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2ff9c522 Branch: refs/heads/master Commit: 2ff9c5229d1470afab12258862f801cf14dc2a2d Parents: c29038a Author: Nishant <nishant.mon...@gmail.com> Authored: Sat Oct 6 02:46:25 2018 +0530 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Tue Oct 9 19:56:06 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/Constants.java | 1 + .../org/apache/hadoop/hive/conf/HiveConf.java | 1 + .../hive/druid/DruidStorageHandlerUtils.java | 4 ++++ .../clientpositive/druidmini_test_insert.q | 9 +++++++ .../druid/druidmini_test_insert.q.out | 25 ++++++++++++++++++++ 5 files changed, 40 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/common/src/java/org/apache/hadoop/hive/conf/Constants.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index 4badfa3..1190679 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -32,6 +32,7 @@ public class Constants { "org.apache.hadoop.hive.druid.io.DruidOutputFormat"; public static final String DRUID_DATA_SOURCE = "druid.datasource"; public static final String DRUID_SEGMENT_GRANULARITY = "druid.segment.granularity"; + public static final String DRUID_ROLLUP = "druid.rollup"; public static final String DRUID_QUERY_GRANULARITY = "druid.query.granularity"; public static final String DRUID_TARGET_SHARDS_PER_GRANULARITY = "druid.segment.targetShardsPerGranularity"; http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d0adc35..cc6239c 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2755,6 +2755,7 @@ public class HiveConf extends Configuration { HIVE_DRUID_BASE_PERSIST_DIRECTORY("hive.druid.basePersistDirectory", "", "Local temporary directory used to persist intermediate indexing state, will default to JVM system property java.io.tmpdir." ), + HIVE_DRUID_ROLLUP("hive.druid.rollup", true, "Whether to rollup druid rows or not."), DRUID_SEGMENT_DIRECTORY("hive.druid.storage.storageDirectory", "/druid/segments" , "druid deep storage location."), DRUID_METADATA_BASE("hive.druid.metadata.base", "druid", "Default prefix for metadata tables"), http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java index e67de89..c3e7e5d 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java @@ -827,12 +827,16 @@ public final class DruidStorageHandlerUtils { tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) != null ? tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) : HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_DRUID_INDEXING_GRANULARITY); + final boolean rollup = tableProperties.getProperty(Constants.DRUID_ROLLUP) != null ? + Boolean.parseBoolean(tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY)): + HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_DRUID_ROLLUP); return new UniformGranularitySpec( Granularity.fromString(segmentGranularity), Granularity.fromString( tableProperties.getProperty(Constants.DRUID_QUERY_GRANULARITY) == null ? "NONE" : tableProperties.getProperty(Constants.DRUID_QUERY_GRANULARITY)), + rollup, null ); } http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/ql/src/test/queries/clientpositive/druidmini_test_insert.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_test_insert.q b/ql/src/test/queries/clientpositive/druidmini_test_insert.q index e97fe5b..28fa6d7 100644 --- a/ql/src/test/queries/clientpositive/druidmini_test_insert.q +++ b/ql/src/test/queries/clientpositive/druidmini_test_insert.q @@ -141,6 +141,15 @@ EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:5 EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp ); EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp ); +-- disable rollup and insert identical rows +SET hive.druid.rollup=false; +insert into druid_test_table_n9 values +('2015-03-12 00:00:00', 'i4-1', 4), +('2015-03-12 00:00:00', 'i4-1', 2), +('2015-03-12 00:00:00', 'i4-1', 1); + +select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ); + DROP TABLE test_base_table; DROP TABLE druid_test_table_n9; http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out index e568b14..b74d33c 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out @@ -824,6 +824,31 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 ListSink +PREHOOK: query: insert into druid_test_table_n9 values +('2015-03-12 00:00:00', 'i4-1', 4), +('2015-03-12 00:00:00', 'i4-1', 2), +('2015-03-12 00:00:00', 'i4-1', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: druid_test_dst@druid_test_table_n9 +POSTHOOK: query: insert into druid_test_table_n9 values +('2015-03-12 00:00:00', 'i4-1', 4), +('2015-03-12 00:00:00', 'i4-1', 2), +('2015-03-12 00:00:00', 'i4-1', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: druid_test_dst@druid_test_table_n9 +PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ) +PREHOOK: type: QUERY +PREHOOK: Input: druid_test_dst@druid_test_table_n9 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp ) +POSTHOOK: type: QUERY +POSTHOOK: Input: druid_test_dst@druid_test_table_n9 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2015-03-12 00:00:00.0 US/Pacific i4-1 4.0 +2015-03-12 00:00:00.0 US/Pacific i4-1 2.0 +2015-03-12 00:00:00.0 US/Pacific i4-1 1.0 PREHOOK: query: DROP TABLE test_base_table PREHOOK: type: DROPTABLE PREHOOK: Input: druid_test_dst@test_base_table