Repository: hive Updated Branches: refs/heads/branch-3 8f4bf93a3 -> 96b5ceb62
HIVE-20184: Backport HIVE-20085 to branch-3 (Daniel Voros via Zoltan Haindrich) Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/96b5ceb6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/96b5ceb6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/96b5ceb6 Branch: refs/heads/branch-3 Commit: 96b5ceb62aaead46830a7440cf12cdd0979b0bba Parents: 8f4bf93 Author: Daniel Voros <daniel.vo...@gmail.com> Authored: Fri Jul 27 13:32:53 2018 +0200 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Fri Jul 27 13:32:53 2018 +0200 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 + .../hadoop/hive/druid/DruidStorageHandler.java | 71 ++++++-------------- .../hadoop/hive/druid/serde/DruidSerDe.java | 3 +- .../hive/druid/TestDruidStorageHandler.java | 4 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +- ql/src/test/queries/clientnegative/ctas.q | 2 +- .../queries/clientnegative/druid_datasource2.q | 1 + .../queries/clientpositive/druid_timestamptz.q | 5 +- .../queries/clientpositive/druid_timestamptz2.q | 14 ++-- .../clientpositive/druidkafkamini_basic.q | 2 +- .../druidmini_dynamic_partition.q | 10 +-- .../clientpositive/druidmini_expressions.q | 3 +- .../clientpositive/druidmini_extractTime.q | 6 +- .../clientpositive/druidmini_floorTime.q | 4 +- .../queries/clientpositive/druidmini_joins.q | 5 +- .../queries/clientpositive/druidmini_masking.q | 4 +- .../queries/clientpositive/druidmini_test1.q | 5 +- .../clientpositive/druidmini_test_alter.q | 5 +- .../clientpositive/druidmini_test_insert.q | 8 ++- .../clientnegative/druid_datasource2.q.out | 2 +- .../druid/druid_timestamptz.q.out | 4 +- .../druid/druid_timestamptz2.q.out | 20 +++--- .../druid/druidkafkamini_basic.q.out | 6 +- .../druid/druidmini_dynamic_partition.q.out | 18 ++--- .../druid/druidmini_expressions.q.out | 6 +- .../druid/druidmini_extractTime.q.out | 8 +-- .../druid/druidmini_floorTime.q.out | 4 +- .../clientpositive/druid/druidmini_joins.q.out | 4 +- .../druid/druidmini_masking.q.out | 4 +- .../clientpositive/druid/druidmini_test1.q.out | 4 +- .../druid/druidmini_test_alter.q.out | 4 +- .../druid/druidmini_test_insert.q.out | 12 ++-- .../hive/metastore/utils/MetaStoreUtils.java | 19 ++++++ 33 files changed, 143 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6c2d0ca..89171ef 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2966,6 +2966,8 @@ public class HiveConf extends Configuration { HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false, "Where to insert into multilevel directories like\n" + "\"insert directory '/HIVEFT25686/chinna/' from table\""), + HIVE_CTAS_EXTERNAL_TABLES("hive.ctas.external.tables", true, + "whether CTAS for external tables is allowed"), HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true, "whether insert into external tables is allowed"), HIVE_TEMPORARY_TABLE_STORAGE( http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java index 57e4800..53d93e1 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java @@ -215,12 +215,10 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor @Override public void preCreateTable(Table table) throws MetaException { - // Do safety checks - if (MetaStoreUtils.isExternalTable(table) && !StringUtils - .isEmpty(table.getSd().getLocation())) { + if(!StringUtils + .isEmpty(table.getSd().getLocation())) { throw new MetaException("LOCATION may not be specified for Druid"); } - if (table.getPartitionKeysSize() != 0) { throw new MetaException("PARTITIONED BY may not be specified for Druid"); } @@ -228,25 +226,17 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor throw new MetaException("CLUSTERED BY may not be specified for Druid"); } String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE); - if (MetaStoreUtils.isExternalTable(table)) { - if (dataSourceName == null) { - throw new MetaException( - String.format("Datasource name should be specified using [%s] for external tables " - + "using Druid", Constants.DRUID_DATA_SOURCE)); - } - // If it is an external table, we are done + if(dataSourceName != null){ + // Already Existing datasource in Druid. return; } - // It is not an external table - // We need to check that datasource was not specified by user - if (dataSourceName != null) { - throw new MetaException( - String.format("Datasource name cannot be specified using [%s] for managed tables " - + "using Druid", Constants.DRUID_DATA_SOURCE)); - } - // We need to check the Druid metadata + + // create dataSourceName based on Hive Table name dataSourceName = Warehouse.getQualifiedName(table); try { + // NOTE: This just created druid_segments table in Druid metastore. + // This is needed for the case when hive is started before any of druid services + // and druid_segments table has not been created yet. getConnector().createSegmentTable(); } catch (Exception e) { LOG.error("Exception while trying to create druid segments table", e); @@ -255,6 +245,7 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor Collection<String> existingDataSources = DruidStorageHandlerUtils .getAllDataSourceNames(getConnector(), getDruidMetadataStorageTablesConfig()); LOG.debug("pre-create data source with name {}", dataSourceName); + // Check for existence of for the datasource we are going to create in druid_segments table. if (existingDataSources.contains(dataSourceName)) { throw new MetaException(String.format("Data source [%s] already existing", dataSourceName)); } @@ -263,38 +254,17 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor @Override public void rollbackCreateTable(Table table) { - if (MetaStoreUtils.isExternalTable(table)) { - return; - } - final Path segmentDescriptorDir = getSegmentDescriptorDir(); - try { - List<DataSegment> dataSegmentList = DruidStorageHandlerUtils - .getCreatedSegments(segmentDescriptorDir, getConf()); - for (DataSegment dataSegment : dataSegmentList) { - try { - deleteSegment(dataSegment); - } catch (SegmentLoadingException e) { - LOG.error(String.format("Error while trying to clean the segment [%s]", dataSegment), e); - } - } - } catch (IOException e) { - LOG.error("Exception while rollback", e); - throw Throwables.propagate(e); - } finally { - cleanWorkingDir(); - } + cleanWorkingDir(); } @Override public void commitCreateTable(Table table) throws MetaException { - if (MetaStoreUtils.isExternalTable(table)) { - // For external tables, we do not need to do anything else - return; - } if(isKafkaStreamingTable(table)){ updateKafkaIngestion(table); } - this.commitInsertTable(table, true); + // For CTAS queries when user has explicitly specified the datasource. + // We will append the data to existing druid datasource. + this.commitInsertTable(table, false); } private void updateKafkaIngestion(Table table){ @@ -762,9 +732,6 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor @Override public void commitDropTable(Table table, boolean deleteData) { - if (MetaStoreUtils.isExternalTable(table)) { - return; - } if(isKafkaStreamingTable(table)) { // Stop Kafka Ingestion first final String overlordAddress = Preconditions.checkNotNull(HiveConf @@ -775,12 +742,15 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor "Druid Datasource name is null"); stopKafkaIngestion(overlordAddress, dataSourceName); } + String dataSourceName = Preconditions .checkNotNull(table.getParameters().get(Constants.DRUID_DATA_SOURCE), "DataSource name is null !" ); - - if (deleteData == true) { + // TODO: Move MetaStoreUtils.isExternalTablePurge(table) calls to a common place for all StorageHandlers + // deleteData flag passed down to StorageHandler should be true only if + // MetaStoreUtils.isExternalTablePurge(table) returns true. + if (deleteData == true && MetaStoreUtils.isExternalTablePurge(table)) { LOG.info("Dropping with purge all the data for data source {}", dataSourceName); List<DataSegment> dataSegmentList = DruidStorageHandlerUtils .getDataSegmentList(getConnector(), getDruidMetadataStorageTablesConfig(), dataSourceName); @@ -806,9 +776,6 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor public void commitInsertTable(Table table, boolean overwrite) throws MetaException { LOG.debug("commit insert into table {} overwrite {}", table.getTableName(), overwrite); - if (MetaStoreUtils.isExternalTable(table)) { - throw new MetaException("Cannot insert data into external table backed by Druid"); - } try { // Check if there segments to load final Path segmentDescriptorDir = getSegmentDescriptorDir(); http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java index f0e12a2..df9049e 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java @@ -483,7 +483,6 @@ import static org.joda.time.format.ISODateTimeFormat.dateOptionalTimeParser; } @Override public boolean shouldStoreFieldsInMetastore(Map<String, String> tableParams) { - // If Druid table is not an external table store the schema in metadata store. - return !MetaStoreUtils.isExternal(tableParams); + return true; } } http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java index b96a13f..510330d 100644 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java +++ b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java @@ -106,7 +106,9 @@ public class TestDruidStorageHandler { public void before() throws Throwable { tableWorkingPath = temporaryFolder.newFolder().getAbsolutePath(); segmentsTable = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(); - Mockito.when(tableMock.getParameters()).thenReturn(new HashMap<>()); + HashMap<String, String> params = new HashMap<>(); + params.put("external.table.purge", "TRUE"); + Mockito.when(tableMock.getParameters()).thenReturn(params); Mockito.when(tableMock.getPartitionKeysSize()).thenReturn(0); StorageDescriptor storageDes = Mockito.mock(StorageDescriptor.class); Mockito.when(storageDes.getBucketColsSize()).thenReturn(0); http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 2bd6c5f..a804c15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -13070,7 +13070,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { throw new SemanticException(ErrorMsg.CTAS_PARCOL_COEXISTENCE.getMsg()); } } - if (isExt) { + if (!conf.getBoolVar(ConfVars.HIVE_CTAS_EXTERNAL_TABLES) && isExt) { throw new SemanticException(ErrorMsg.CTAS_EXTTBL_COEXISTENCE.getMsg()); } command_type = CTAS; http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientnegative/ctas.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/ctas.q b/ql/src/test/queries/clientnegative/ctas.q index 507a7a7..1d0afaf 100644 --- a/ql/src/test/queries/clientnegative/ctas.q +++ b/ql/src/test/queries/clientnegative/ctas.q @@ -1,5 +1,5 @@ - +SET hive.ctas.external.tables=false; create external table nzhang_ctas4 as select key, value from src; http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientnegative/druid_datasource2.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/druid_datasource2.q b/ql/src/test/queries/clientnegative/druid_datasource2.q index cc20931..2e7400f 100644 --- a/ql/src/test/queries/clientnegative/druid_datasource2.q +++ b/ql/src/test/queries/clientnegative/druid_datasource2.q @@ -1,3 +1,4 @@ +SET metastore.strict.managed.tables=true; CREATE TABLE druid_table_1 STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler' TBLPROPERTIES ("property" = "localhost", "druid.datasource" = "mydatasource"); http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druid_timestamptz.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz.q b/ql/src/test/queries/clientpositive/druid_timestamptz.q index 4830044..605d240 100644 --- a/ql/src/test/queries/clientpositive/druid_timestamptz.q +++ b/ql/src/test/queries/clientpositive/druid_timestamptz.q @@ -1,9 +1,10 @@ set hive.fetch.task.conversion=more; - +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; drop table tstz1_n0; -create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) +create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR"); http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druid_timestamptz2.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz2.q b/ql/src/test/queries/clientpositive/druid_timestamptz2.q index 8f573c8..8b2c092 100644 --- a/ql/src/test/queries/clientpositive/druid_timestamptz2.q +++ b/ql/src/test/queries/clientpositive/druid_timestamptz2.q @@ -1,9 +1,9 @@ +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; CREATE database druid_test_dst; use druid_test_dst; - - -create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double); +create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double); insert into test_base_table values ('2015-03-08 00:00:00', 'i1-start', 4); insert into test_base_table values ('2015-03-08 23:59:59', 'i1-end', 1); insert into test_base_table values ('2015-03-09 00:00:00', 'i2-start', 4); @@ -11,7 +11,7 @@ insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1); insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2); insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2); -CREATE TABLE druid_test_table_1 +CREATE EXTERNAL TABLE druid_test_table_1 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -20,7 +20,7 @@ FROM druid_test_dst.test_base_table; select * FROM druid_test_table_1; -CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY"); @@ -36,7 +36,7 @@ select * FROM druid_test_table_2; SET TIME ZONE UTC; -CREATE TABLE druid_test_table_utc +CREATE EXTERNAL TABLE druid_test_table_utc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -45,7 +45,7 @@ FROM druid_test_dst.test_base_table; select * FROM druid_test_table_utc; -CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY"); http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidkafkamini_basic.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidkafkamini_basic.q b/ql/src/test/queries/clientpositive/druidkafkamini_basic.q index 814890a..4cdabf2 100644 --- a/ql/src/test/queries/clientpositive/druidkafkamini_basic.q +++ b/ql/src/test/queries/clientpositive/druidkafkamini_basic.q @@ -1,5 +1,5 @@ SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) +CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "MONTH", http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q b/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q index 4443af2..de42900 100644 --- a/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q +++ b/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q @@ -1,5 +1,7 @@ SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_partitioned_table_0 +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_partitioned_table_0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -20,7 +22,7 @@ CREATE TABLE druid_partitioned_table_0 cboolean2 FROM alltypesorc where ctimestamp1 IS NOT NULL; -EXPLAIN CREATE TABLE druid_partitioned_table +EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -43,7 +45,7 @@ EXPLAIN CREATE TABLE druid_partitioned_table -CREATE TABLE druid_partitioned_table +CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -154,7 +156,7 @@ SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table ; set hive.druid.indexer.partition.size.max=10; -CREATE TABLE druid_max_size_partition +CREATE EXTERNAL TABLE druid_max_size_partition STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_expressions.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_expressions.q b/ql/src/test/queries/clientpositive/druidmini_expressions.q index fad8f73..7857973 100644 --- a/ql/src/test/queries/clientpositive/druidmini_expressions.q +++ b/ql/src/test/queries/clientpositive/druidmini_expressions.q @@ -1,7 +1,8 @@ --! qt:dataset:alltypesorc +SET hive.ctas.external.tables=true; SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_table_n0 +CREATE EXTERNAL TABLE druid_table_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_extractTime.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_extractTime.q b/ql/src/test/queries/clientpositive/druidmini_extractTime.q index 9541361..0dbcd25 100644 --- a/ql/src/test/queries/clientpositive/druidmini_extractTime.q +++ b/ql/src/test/queries/clientpositive/druidmini_extractTime.q @@ -1,7 +1,9 @@ --! qt:dataset:alltypesorc SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_table +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -181,7 +183,7 @@ SELECT CAST(`__time` AS DATE) AS `x_date` FROM druid_table ORDER BY `x_date` LIM create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double); insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0); -CREATE TABLE druid_test_extract_from_string_table +CREATE EXTERNAL TABLE druid_test_extract_from_string_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS select http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_floorTime.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_floorTime.q b/ql/src/test/queries/clientpositive/druidmini_floorTime.q index a526413..3b5334f 100644 --- a/ql/src/test/queries/clientpositive/druidmini_floorTime.q +++ b/ql/src/test/queries/clientpositive/druidmini_floorTime.q @@ -1,7 +1,9 @@ --! qt:dataset:alltypesorc SET hive.vectorized.execution.enabled=false; -CREATE TABLE druid_table_n2 +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_table_n2 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_joins.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_joins.q b/ql/src/test/queries/clientpositive/druidmini_joins.q index 720127e..1f92a0d 100644 --- a/ql/src/test/queries/clientpositive/druidmini_joins.q +++ b/ql/src/test/queries/clientpositive/druidmini_joins.q @@ -1,11 +1,12 @@ SET hive.vectorized.execution.enabled=false; SET hive.explain.user=false; - +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; --SET hive.execution.mode=llap; DROP TABLE druid_table_with_nulls; -CREATE TABLE druid_table_with_nulls +CREATE EXTERNAL TABLE druid_table_with_nulls STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_masking.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_masking.q b/ql/src/test/queries/clientpositive/druidmini_masking.q index f0f2c0c..824611f 100644 --- a/ql/src/test/queries/clientpositive/druidmini_masking.q +++ b/ql/src/test/queries/clientpositive/druidmini_masking.q @@ -1,7 +1,9 @@ set hive.mapred.mode=nonstrict; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; -CREATE TABLE masking_test_druid +CREATE EXTERNAL TABLE masking_test_druid STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_test1.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_test1.q b/ql/src/test/queries/clientpositive/druidmini_test1.q index f93665e..30abf3c 100644 --- a/ql/src/test/queries/clientpositive/druidmini_test1.q +++ b/ql/src/test/queries/clientpositive/druidmini_test1.q @@ -1,5 +1,8 @@ --! qt:dataset:alltypesorc -CREATE TABLE druid_table_n3 +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; + +CREATE EXTERNAL TABLE druid_table_n3 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_test_alter.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_test_alter.q b/ql/src/test/queries/clientpositive/druidmini_test_alter.q index e19a006..0d8623d 100644 --- a/ql/src/test/queries/clientpositive/druidmini_test_alter.q +++ b/ql/src/test/queries/clientpositive/druidmini_test_alter.q @@ -1,4 +1,7 @@ -CREATE TABLE druid_alltypesorc_n0 +--! qt:dataset:alltypesorc +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_alltypesorc_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/queries/clientpositive/druidmini_test_insert.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druidmini_test_insert.q b/ql/src/test/queries/clientpositive/druidmini_test_insert.q index cdd4c23..dd1ba1a 100644 --- a/ql/src/test/queries/clientpositive/druidmini_test_insert.q +++ b/ql/src/test/queries/clientpositive/druidmini_test_insert.q @@ -1,4 +1,6 @@ -CREATE TABLE druid_alltypesorc +SET hive.ctas.external.tables=true; +SET hive.external.table.purge.default = true; +CREATE EXTERNAL TABLE druid_alltypesorc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -62,7 +64,7 @@ DROP TABLE druid_alltypesorc; insert into test_table_n9 values ('2015-01-08 00:00:00', 'i1-start', 4); insert into test_table_n9 values ('2015-01-08 23:59:59', 'i1-end', 1); - CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) + CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY"); @@ -89,7 +91,7 @@ insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1); insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2); insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2); -CREATE TABLE druid_test_table_n9 +CREATE EXTERNAL TABLE druid_test_table_n9 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientnegative/druid_datasource2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/druid_datasource2.q.out b/ql/src/test/results/clientnegative/druid_datasource2.q.out index 1b74b06..2f783fe 100644 --- a/ql/src/test/results/clientnegative/druid_datasource2.q.out +++ b/ql/src/test/results/clientnegative/druid_datasource2.q.out @@ -4,4 +4,4 @@ TBLPROPERTIES ("property" = "localhost", "druid.datasource" = "mydatasource") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Datasource name cannot be specified using [druid.datasource] for managed tables using Druid) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Table default.druid_table_1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.) http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out index ed1a6ea..fa9583a 100644 --- a/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out +++ b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out @@ -2,13 +2,13 @@ PREHOOK: query: drop table tstz1_n0 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table tstz1_n0 POSTHOOK: type: DROPTABLE -PREHOOK: query: create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) +PREHOOK: query: create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tstz1_n0 -POSTHOOK: query: create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) +POSTHOOK: query: create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") POSTHOOK: type: CREATETABLE http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out b/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out index c71a435..b62095a 100644 --- a/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out +++ b/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out @@ -10,11 +10,11 @@ PREHOOK: Input: database:druid_test_dst POSTHOOK: query: use druid_test_dst POSTHOOK: type: SWITCHDATABASE POSTHOOK: Input: database:druid_test_dst -PREHOOK: query: create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) +PREHOOK: query: create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) PREHOOK: type: CREATETABLE PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@test_base_table -POSTHOOK: query: create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) +POSTHOOK: query: create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:druid_test_dst POSTHOOK: Output: druid_test_dst@test_base_table @@ -84,7 +84,7 @@ POSTHOOK: Output: druid_test_dst@test_base_table POSTHOOK: Lineage: test_base_table.interval_marker SCRIPT [] POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] -PREHOOK: query: CREATE TABLE druid_test_table_1 +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_1 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -94,7 +94,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: druid_test_dst@test_base_table PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_1 -POSTHOOK: query: CREATE TABLE druid_test_table_1 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_1 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -121,13 +121,13 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") PREHOOK: type: CREATETABLE PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_2 -POSTHOOK: query: CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") POSTHOOK: type: CREATETABLE @@ -195,7 +195,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 2015-03-09 23:59:59.0 US/Pacific i2-end 1.0 2015-03-10 00:00:00.0 US/Pacific i3-start 2.0 2015-03-10 23:59:59.0 US/Pacific i3-end 2.0 -PREHOOK: query: CREATE TABLE druid_test_table_utc +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -205,7 +205,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: druid_test_dst@test_base_table PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_utc -POSTHOOK: query: CREATE TABLE druid_test_table_utc +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -232,13 +232,13 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 2015-03-09 23:59:59.0 UTC i2-end 1.0 2015-03-10 00:00:00.0 UTC i3-start 2.0 2015-03-10 23:59:59.0 UTC i3-end 2.0 -PREHOOK: query: CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") PREHOOK: type: CREATETABLE PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_utc2 -POSTHOOK: query: CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") POSTHOOK: type: CREATETABLE http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out index 0743974..ffd9279 100644 --- a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out +++ b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) +PREHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "MONTH", @@ -15,7 +15,7 @@ PREHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, ` PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@druid_kafka_test -POSTHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) +POSTHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "MONTH", @@ -339,6 +339,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","deleted":"true","language":"true","page":"true","user":"true"}} + EXTERNAL TRUE bucket_count -1 bucketing_version 2 column.name.delimiter , @@ -379,6 +380,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","deleted":"true","language":"true","page":"true","user":"true"}} + EXTERNAL TRUE bucket_count -1 bucketing_version 2 column.name.delimiter , http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out b/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out index 28deb79..8b1e446 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_partitioned_table_0 +PREHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table_0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -22,7 +22,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_partitioned_table_0 -POSTHOOK: query: CREATE TABLE druid_partitioned_table_0 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table_0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -57,7 +57,7 @@ POSTHOOK: Lineage: druid_partitioned_table_0.csmallint SIMPLE [(alltypesorc)allt POSTHOOK: Lineage: druid_partitioned_table_0.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: druid_partitioned_table_0.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: druid_partitioned_table_0.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table +PREHOOK: query: EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -78,7 +78,7 @@ PREHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table cboolean2 FROM alltypesorc where ctimestamp1 IS NOT NULL PREHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table +POSTHOOK: query: EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -165,6 +165,8 @@ STAGE PLANS: druid.query.granularity MINUTE druid.segment.granularity HOUR druid.segment.targetShardsPerGranularity 6 + external.table.purge true + isExternal: true Stage: Stage-3 Stats Work @@ -176,7 +178,7 @@ STAGE PLANS: hdfs directory: true destination: hdfs://### HDFS PATH ### -PREHOOK: query: CREATE TABLE druid_partitioned_table +PREHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -200,7 +202,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_partitioned_table -POSTHOOK: query: CREATE TABLE druid_partitioned_table +POSTHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -567,7 +569,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@druid_partitioned_table POSTHOOK: Output: hdfs://### HDFS PATH ### 1408069801800 10992545287 -PREHOOK: query: CREATE TABLE druid_max_size_partition +PREHOOK: query: CREATE EXTERNAL TABLE druid_max_size_partition STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", @@ -590,7 +592,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_max_size_partition -POSTHOOK: query: CREATE TABLE druid_max_size_partition +POSTHOOK: query: CREATE EXTERNAL TABLE druid_max_size_partition STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ( "druid.segment.granularity" = "HOUR", http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out index fd77a91..e322d8f 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_table_n0 +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_n0 -POSTHOOK: query: CREATE TABLE druid_table_n0 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -270,6 +270,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + EXTERNAL TRUE bucket_count -1 bucketing_version 2 column.name.delimiter , @@ -301,6 +302,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + EXTERNAL TRUE bucket_count -1 bucketing_version 2 column.name.delimiter , http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out b/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out index 30e273b..116c326 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_table +PREHOOK: query: CREATE EXTERNAL TABLE druid_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_table -POSTHOOK: query: CREATE TABLE druid_table +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -776,7 +776,7 @@ POSTHOOK: Lineage: test_extract_from_string_base_table.date_c SCRIPT [] POSTHOOK: Lineage: test_extract_from_string_base_table.metric_c SCRIPT [] POSTHOOK: Lineage: test_extract_from_string_base_table.timecolumn SCRIPT [] POSTHOOK: Lineage: test_extract_from_string_base_table.timestamp_c SCRIPT [] -PREHOOK: query: CREATE TABLE druid_test_extract_from_string_table +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_extract_from_string_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS select @@ -786,7 +786,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@test_extract_from_string_base_table PREHOOK: Output: database:default PREHOOK: Output: default@druid_test_extract_from_string_table -POSTHOOK: query: CREATE TABLE druid_test_extract_from_string_table +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_extract_from_string_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS select http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out b/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out index c54fd93..1c9e9c6 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_table_n2 +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n2 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_n2 -POSTHOOK: query: CREATE TABLE druid_table_n2 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n2 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out b/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out index 73a3c9f..014c7b5 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out @@ -2,7 +2,7 @@ PREHOOK: query: DROP TABLE druid_table_with_nulls PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE druid_table_with_nulls POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE druid_table_with_nulls +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_with_nulls STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS @@ -22,7 +22,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_with_nulls -POSTHOOK: query: CREATE TABLE druid_table_with_nulls +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_with_nulls STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out b/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out index 1aad967..e3b50a4 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE masking_test_druid +PREHOOK: query: CREATE EXTERNAL TABLE masking_test_druid STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS @@ -17,7 +17,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: database:default PREHOOK: Output: default@masking_test_druid -POSTHOOK: query: CREATE TABLE masking_test_druid +POSTHOOK: query: CREATE EXTERNAL TABLE masking_test_druid STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out index 4e078aa..7f6c6b0 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_table_n3 +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n3 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_table_n3 -POSTHOOK: query: CREATE TABLE druid_table_n3 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n3 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out index 439dc9c..a175f76 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_alltypesorc_n0 +PREHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -16,7 +16,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_alltypesorc_n0 -POSTHOOK: query: CREATE TABLE druid_alltypesorc_n0 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc_n0 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out index c675aeb..0da7610 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out @@ -1,4 +1,4 @@ -PREHOOK: query: CREATE TABLE druid_alltypesorc +PREHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@druid_alltypesorc -POSTHOOK: query: CREATE TABLE druid_alltypesorc +POSTHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE") AS @@ -194,13 +194,13 @@ POSTHOOK: Output: druid_test_create_then_insert@test_table_n9 POSTHOOK: Lineage: test_table_n9.num_l SCRIPT [] POSTHOOK: Lineage: test_table_n9.timecolumn SCRIPT [] POSTHOOK: Lineage: test_table_n9.userid SCRIPT [] -PREHOOK: query: CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) +PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") PREHOOK: type: CREATETABLE PREHOOK: Output: database:druid_test_create_then_insert PREHOOK: Output: druid_test_create_then_insert@druid_table_n1 -POSTHOOK: query: CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) +POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float) STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") POSTHOOK: type: CREATETABLE @@ -335,7 +335,7 @@ POSTHOOK: Output: druid_test_dst@test_base_table POSTHOOK: Lineage: test_base_table.num_l SCRIPT [] POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT [] POSTHOOK: Lineage: test_base_table.userid SCRIPT [] -PREHOOK: query: CREATE TABLE druid_test_table_n9 +PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_n9 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS @@ -344,7 +344,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: druid_test_dst@test_base_table PREHOOK: Output: database:druid_test_dst PREHOOK: Output: druid_test_dst@druid_test_table_n9 -POSTHOOK: query: CREATE TABLE druid_test_table_n9 +POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_n9 STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' TBLPROPERTIES ("druid.segment.granularity" = "DAY") AS http://git-wip-us.apache.org/repos/asf/hive/blob/96b5ceb6/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index cbe89b6..4bc819f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -568,6 +568,25 @@ public class MetaStoreUtils { return isExternal(params); } + /** + * Determines whether an table needs to be purged or not. + * + * @param table table of interest + * + * @return true if external table needs to be purged + */ + public static boolean isExternalTablePurge(Table table) { + if (table == null) { + return false; + } + Map<String, String> params = table.getParameters(); + if (params == null) { + return false; + } + + return isPropertyTrue(params, EXTERNAL_TABLE_PURGE); + } + public static boolean isExternal(Map<String, String> tableParams){ return isPropertyTrue(tableParams, "EXTERNAL"); }