http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index 1cf47c3..50d8878 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -31,13 +31,16 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Stack; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreEventListener; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FireEventRequest; @@ -46,6 +49,7 @@ import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.FunctionType; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; import org.apache.hadoop.hive.metastore.api.Order; @@ -56,6 +60,21 @@ import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; +import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; +import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterTableEvent; +import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent; +import org.apache.hadoop.hive.metastore.events.CreateTableEvent; +import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropFunctionEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; +import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; +import org.apache.hadoop.hive.metastore.events.DropTableEvent; +import org.apache.hadoop.hive.metastore.events.InsertEvent; +import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.AlterIndexMessage; import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; @@ -75,6 +94,8 @@ import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer; import org.apache.hadoop.hive.metastore.messaging.MessageFactory; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.data.Pair; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -97,12 +118,105 @@ public class TestDbNotificationListener { private int startTime; private long firstEventId; + /* This class is used to verify that HiveMetaStore calls the non-transactional listeners with the + * current event ID set by the DbNotificationListener class */ + public static class MockMetaStoreEventListener extends MetaStoreEventListener { + private static Stack<Pair<EventType, String>> eventsIds = new Stack<>(); + + private static void pushEventId(EventType eventType, final ListenerEvent event) { + if (event.getStatus()) { + Map<String, String> parameters = event.getParameters(); + if (parameters.containsKey(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) { + Pair<EventType, String> pair = + new Pair<>(eventType, parameters.get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)); + eventsIds.push(pair); + } + } + } + + public static void popAndVerifyLastEventId(EventType eventType, long id) { + if (!eventsIds.isEmpty()) { + Pair<EventType, String> pair = eventsIds.pop(); + + assertEquals("Last event type does not match.", eventType, pair.first); + assertEquals("Last event ID does not match.", Long.toString(id), pair.second); + } else { + assertTrue("List of events is empty.",false); + } + } + + public static void clearEvents() { + eventsIds.clear(); + } + + public MockMetaStoreEventListener(Configuration config) { + super(config); + } + + public void onCreateTable (CreateTableEvent tableEvent) throws MetaException { + pushEventId(EventType.CREATE_TABLE, tableEvent); + } + + public void onDropTable (DropTableEvent tableEvent) throws MetaException { + pushEventId(EventType.DROP_TABLE, tableEvent); + } + + public void onAlterTable (AlterTableEvent tableEvent) throws MetaException { + pushEventId(EventType.ALTER_TABLE, tableEvent); + } + + public void onAddPartition (AddPartitionEvent partitionEvent) throws MetaException { + pushEventId(EventType.ADD_PARTITION, partitionEvent); + } + + public void onDropPartition (DropPartitionEvent partitionEvent) throws MetaException { + pushEventId(EventType.DROP_PARTITION, partitionEvent); + } + + public void onAlterPartition (AlterPartitionEvent partitionEvent) throws MetaException { + pushEventId(EventType.ALTER_PARTITION, partitionEvent); + } + + public void onCreateDatabase (CreateDatabaseEvent dbEvent) throws MetaException { + pushEventId(EventType.CREATE_DATABASE, dbEvent); + } + + public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException { + pushEventId(EventType.DROP_DATABASE, dbEvent); + } + + public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { + pushEventId(EventType.CREATE_INDEX, indexEvent); + } + + public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { + pushEventId(EventType.DROP_INDEX, indexEvent); + } + + public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { + pushEventId(EventType.ALTER_INDEX, indexEvent); + } + + public void onCreateFunction (CreateFunctionEvent fnEvent) throws MetaException { + pushEventId(EventType.CREATE_FUNCTION, fnEvent); + } + + public void onDropFunction (DropFunctionEvent fnEvent) throws MetaException { + pushEventId(EventType.DROP_FUNCTION, fnEvent); + } + + public void onInsert(InsertEvent insertEvent) throws MetaException { + pushEventId(EventType.INSERT, insertEvent); + } + } + @SuppressWarnings("rawtypes") @BeforeClass public static void connectToMetastore() throws Exception { HiveConf conf = new HiveConf(); conf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, DbNotificationListener.class.getName()); + conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, MockMetaStoreEventListener.class.getName()); conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL, String.valueOf(EVENTS_TTL) + "s"); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true); @@ -139,6 +253,12 @@ public class TestDbNotificationListener { DummyRawStoreFailEvent.setEventSucceed(true); } + @After + public void tearDown() { + MockMetaStoreEventListener.clearEvents(); + } + + @Test public void createDatabase() throws Exception { String dbName = "createdb"; @@ -164,6 +284,9 @@ public class TestDbNotificationListener { CreateDatabaseMessage createDbMsg = md.getCreateDatabaseMessage(event.getMessage()); assertEquals(dbName, createDbMsg.getDB()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_DATABASE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification DummyRawStoreFailEvent.setEventSucceed(false); @@ -206,6 +329,10 @@ public class TestDbNotificationListener { DropDatabaseMessage dropDbMsg = md.getDropDatabaseMessage(event.getMessage()); assertEquals(dbName, dropDbMsg.getDB()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_DATABASE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_DATABASE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification db = new Database(dbName2, dbDescription, dbLocationUri, emptyParameters); @@ -256,6 +383,9 @@ public class TestDbNotificationListener { assertEquals(tblName, createTblMsg.getTable()); assertEquals(table, createTblMsg.getTableObj()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification table = @@ -312,6 +442,9 @@ public class TestDbNotificationListener { AlterTableMessage alterTableMessage = md.getAlterTableMessage(event.getMessage()); assertEquals(table, alterTableMessage.getTableObjAfter()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification DummyRawStoreFailEvent.setEventSucceed(false); @@ -363,6 +496,10 @@ public class TestDbNotificationListener { assertEquals(defaultDbName, dropTblMsg.getDB()); assertEquals(tblName, dropTblMsg.getTable()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification table = @@ -428,6 +565,10 @@ public class TestDbNotificationListener { assertTrue(ptnIter.hasNext()); assertEquals(partition, ptnIter.next()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification partition = @@ -494,6 +635,10 @@ public class TestDbNotificationListener { assertEquals(tblName, alterPtnMsg.getTable()); assertEquals(newPart, alterPtnMsg.getPtnObjAfter()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification DummyRawStoreFailEvent.setEventSucceed(false); @@ -557,6 +702,11 @@ public class TestDbNotificationListener { assertEquals(table.getTableName(), tableObj.getTableName()); assertEquals(table.getOwner(), tableObj.getOwner()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification List<String> newpartCol1Vals = Arrays.asList("tomorrow"); @@ -653,6 +803,13 @@ public class TestDbNotificationListener { Iterator<Map<String, String>> parts = dropPtnMsg.getPartitions().iterator(); assertTrue(parts.hasNext()); assertEquals(part1.getValues(), Lists.newArrayList(parts.next().values())); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 5); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 4); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); } @Test @@ -693,6 +850,9 @@ public class TestDbNotificationListener { assertEquals(ResourceType.JAR, funcObj.getResourceUris().get(0).getResourceType()); assertEquals(funcResource, funcObj.getResourceUris().get(0).getUri()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_FUNCTION, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification DummyRawStoreFailEvent.setEventSucceed(false); @@ -742,6 +902,10 @@ public class TestDbNotificationListener { assertEquals(defaultDbName, dropFuncMsg.getDB()); assertEquals(funcName, dropFuncMsg.getFunctionName()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_FUNCTION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_FUNCTION, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification func = @@ -807,6 +971,11 @@ public class TestDbNotificationListener { assertEquals(tableName, indexObj.getOrigTableName()); assertEquals(indexTableName, indexObj.getIndexTableName()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification DummyRawStoreFailEvent.setEventSucceed(false); @@ -873,6 +1042,12 @@ public class TestDbNotificationListener { assertEquals(indexTableName.toLowerCase(), dropIdxMsg.getIndexTableName()); assertEquals(tableName.toLowerCase(), dropIdxMsg.getOrigTableName()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_INDEX, firstEventId + 4); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification index = @@ -947,6 +1122,12 @@ public class TestDbNotificationListener { assertEquals(indexTableName, indexObj.getIndexTableName()); assertTrue(indexObj.getCreateTime() < indexObj.getLastAccessTime()); + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ALTER_INDEX, firstEventId + 4); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); + // When hive.metastore.transactional.event.listeners is set, // a failed event should not create a new notification DummyRawStoreFailEvent.setEventSucceed(false); @@ -1003,6 +1184,10 @@ public class TestDbNotificationListener { assertEquals(tblName, event.getTableName()); // Parse the message field verifyInsert(event, defaultDbName, tblName); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); } @Test @@ -1063,6 +1248,11 @@ public class TestDbNotificationListener { Map<String,String> partKeyValsFromNotif = insertMessage.getPartitionKeyValues(); assertMapEquals(partKeyVals, partKeyValsFromNotif); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); }
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/pom.xml ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/pom.xml b/itests/hive-blobstore/pom.xml index b18398d..d1c732d 100644 --- a/itests/hive-blobstore/pom.xml +++ b/itests/hive-blobstore/pom.xml @@ -20,7 +20,7 @@ <parent> <groupId>org.apache.hive</groupId> <artifactId>hive-it</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>3.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_blobstore.q new file mode 100644 index 0000000..8fee8ed --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_blobstore.q @@ -0,0 +1,45 @@ +-- Check we can create a partitioned table in the warehouse, +-- export it to a blobstore, and then import its different partitions +-- using the blobstore as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz"); + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/export/exim_employee; +EXPORT TABLE exim_employee +TO '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/export/exim_employee'; + +DROP TABLE exim_employee; +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/import/exim_employee; +IMPORT TABLE exim_employee PARTITION (emp_country='us') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/export/exim_employee' +LOCATION '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +IMPORT TABLE exim_employee PARTITION (emp_country='cz') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/export/exim_employee' +LOCATION '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +IMPORT TABLE exim_employee PARTITION (emp_country='in') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/export/exim_employee' +LOCATION '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_blobstore/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_local.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_local.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_local.q new file mode 100644 index 0000000..28bc399 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_local.q @@ -0,0 +1,44 @@ +-- Check we can create a partitioned table in the warehouse, +-- export it to a blobstore, and then import its different partitions +-- using the local filesystem as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz"); + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_local/export/exim_employee; +EXPORT TABLE exim_employee +TO '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_local/export/exim_employee'; + +DROP TABLE exim_employee; +dfs -rm -r -f ${system:build.test.dir}/import_addpartition_blobstore_to_local/import/exim_employee; +IMPORT TABLE exim_employee PARTITION (emp_country='us') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_local/export/exim_employee' +LOCATION 'file://${system:build.test.dir}/import_addpartition_blobstore_to_local/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +IMPORT TABLE exim_employee PARTITION (emp_country='cz') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_local/export/exim_employee' +LOCATION 'file://${system:build.test.dir}/import_addpartition_blobstore_to_local/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +IMPORT TABLE exim_employee PARTITION (emp_country='in') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_local/export/exim_employee' +LOCATION 'file://${system:build.test.dir}/import_addpartition_blobstore_to_local/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_warehouse.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_warehouse.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_warehouse.q new file mode 100644 index 0000000..987dacf --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_blobstore_to_warehouse.q @@ -0,0 +1,41 @@ +-- Check we can create a partitioned table in the warehouse, +-- export it to a blobstore, and then import its different partitions +-- using the warehouse as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz"); + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_warehouse/export/exim_employee; +EXPORT TABLE exim_employee +TO '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_warehouse/export/exim_employee'; + +DROP TABLE exim_employee; +IMPORT TABLE exim_employee PARTITION (emp_country='us') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_warehouse/export/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +IMPORT TABLE exim_employee PARTITION (emp_country='cz') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_warehouse/export/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +IMPORT TABLE exim_employee PARTITION (emp_country='in') +FROM '${hiveconf:test.blobstore.path.unique}/import_addpartition_blobstore_to_warehouse/export/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_local_to_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_local_to_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_local_to_blobstore.q new file mode 100644 index 0000000..8fde250 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_addpartition_local_to_blobstore.q @@ -0,0 +1,44 @@ +-- Check we can create a partitioned table in the warehouse, +-- export it to the local filesystem, and then import its +-- different partitions using a blobstore as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz"); + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${system:build.test.dir}/import_addpartition_local_to_blobstore/export/exim_employee; +EXPORT TABLE exim_employee +TO 'file://${system:build.test.dir}/import_addpartition_local_to_blobstore/export/exim_employee'; + +DROP TABLE exim_employee; +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_addpartition_local_to_blobstore/import/exim_employee; +IMPORT TABLE exim_employee PARTITION (emp_country='us') +FROM 'file://${system:build.test.dir}/import_addpartition_local_to_blobstore/export/exim_employee' +LOCATION '${hiveconf:test.blobstore.path.unique}/import_addpartition_local_to_blobstore/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +IMPORT TABLE exim_employee PARTITION (emp_country='cz') +FROM 'file://${system:build.test.dir}/import_addpartition_local_to_blobstore/export/exim_employee' +LOCATION '${hiveconf:test.blobstore.path.unique}/import_addpartition_local_to_blobstore/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +IMPORT TABLE exim_employee PARTITION (emp_country='in') +FROM 'file://${system:build.test.dir}/import_addpartition_local_to_blobstore/export/exim_employee' +LOCATION '${hiveconf:test.blobstore.path.unique}/import_addpartition_local_to_blobstore/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_blobstore.q new file mode 100644 index 0000000..a9f9a8f --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_blobstore.q @@ -0,0 +1,30 @@ +-- Check we can create a partitioned table in the warehouse, +-- export it to a blobstore, and then import the +-- whole table using the blobstore as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz"); + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore/export/exim_employee; +EXPORT TABLE exim_employee PARTITION (emp_country='us') +TO '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore/export/exim_employee'; + +DROP TABLE exim_employee; +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore/import/exim_employee; +IMPORT FROM '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore/export/exim_employee' +LOCATION '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_blobstore_nonpart.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_blobstore_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_blobstore_nonpart.q new file mode 100644 index 0000000..7b3f0a3 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_blobstore_nonpart.q @@ -0,0 +1,25 @@ +-- Check we can create a non partitioned table in the warehouse, +-- export it to a blobstore, and then import the +-- table using the blobstore as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore_nonpart/export/exim_employee; +EXPORT TABLE exim_employee +TO '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore_nonpart/export/exim_employee'; + +DROP TABLE exim_employee; +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore_nonpart/import/exim_employee; +IMPORT FROM '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore_nonpart/export/exim_employee' +LOCATION '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_blobstore_nonpart/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_local.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_local.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_local.q new file mode 100644 index 0000000..ac3c451 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_local.q @@ -0,0 +1,30 @@ +-- Check we can create a partitioned table in the warehouse, +-- export it to a blobstore, and then import the +-- whole table using the local filesystem as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz"); + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_blobstore_to_local/export/exim_employee; +EXPORT TABLE exim_employee PARTITION (emp_country='us') +TO '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_local/export/exim_employee'; + +DROP TABLE exim_employee; +dfs -rm -r -f ${system:build.test.dir}/import_blobstore_to_local/import/exim_employee; +IMPORT FROM '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_local/export/exim_employee' +LOCATION "file://${system:build.test.dir}/import_blobstore_to_local/import/exim_employee"; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_warehouse.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_warehouse.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_warehouse.q new file mode 100644 index 0000000..9f6fc54 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_warehouse.q @@ -0,0 +1,28 @@ +-- Check we can create a partitioned table in the warehouse, +-- export it to a blobstore, and then import the +-- whole table using the warehouse as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz"); + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_blobstore_to_warehouse/export/exim_employee; +EXPORT TABLE exim_employee PARTITION (emp_country='us') +TO '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_warehouse/export/exim_employee'; + +DROP TABLE exim_employee; +IMPORT FROM '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_warehouse/export/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_warehouse_nonpart.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_warehouse_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_warehouse_nonpart.q new file mode 100644 index 0000000..6f28a51 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_blobstore_to_warehouse_nonpart.q @@ -0,0 +1,23 @@ +-- Check we can create a non partitioned table in the warehouse, +-- export it to a blobstore, and then import the +-- table using the warehouse as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_blobstore_to_warehouse_nonpart/export/exim_employee; +EXPORT TABLE exim_employee +TO '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_warehouse_nonpart/export/exim_employee'; + +DROP TABLE exim_employee; +IMPORT FROM '${hiveconf:test.blobstore.path.unique}/import_blobstore_to_warehouse_nonpart/export/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/import_local_to_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/import_local_to_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/import_local_to_blobstore.q new file mode 100644 index 0000000..0412d3c --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/import_local_to_blobstore.q @@ -0,0 +1,31 @@ +-- Check we can create a partitioned table in the warehouse, +-- export it to the local filesystem, and then import the +-- whole table using a blobstore as target location +DROP TABLE exim_employee; +CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us"); +LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz"); + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; + +dfs -rm -r -f ${system:build.test.dir}/import_local_to_blobstore/export/exim_employee; + +EXPORT TABLE exim_employee PARTITION (emp_country='us') +TO 'file://${system:build.test.dir}/import_local_to_blobstore/export/exim_employee'; + +DROP TABLE exim_employee; +dfs -rm -r -f ${hiveconf:test.blobstore.path.unique}/import_local_to_blobstore/import/exim_employee; +IMPORT FROM "file://${system:build.test.dir}/import_local_to_blobstore/export/exim_employee" +LOCATION '${hiveconf:test.blobstore.path.unique}/import_local_to_blobstore/import/exim_employee'; + +DESCRIBE EXTENDED exim_employee; +SELECT * FROM exim_employee; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q new file mode 100644 index 0000000..8219ee2 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q @@ -0,0 +1,29 @@ +-- Test inserting into a blobstore table from another blobstore table. + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source ( + a string, + b string, + c double) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/insert_blobstore_to_blobstore/blobstore_source'; + +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE blobstore_table; +CREATE TABLE blobstore_table LIKE blobstore_source +LOCATION '${hiveconf:test.blobstore.path.unique}/insert_blobstore_to_blobstore/blobstore_table'; + +INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source; + +SELECT COUNT(*) FROM blobstore_table; + +-- INSERT INTO should append all records to existing ones. +INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source; + +SELECT COUNT(*) FROM blobstore_table; + +SELECT * FROM blobstore_table; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q new file mode 100644 index 0000000..d4f0c71 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q @@ -0,0 +1,53 @@ +-- Test inserting empty rows into dynamic partitioned and list bucketed blobstore tables + +SET hive.blobstore.optimizations.enabled=true; + +DROP TABLE empty; +DROP TABLE blobstore_dynamic_partitioning; +DROP TABLE blobstore_list_bucketing; + +CREATE TABLE empty ( + id int, + name string, + dept string, + pt string, + dt string, + hr string); + +CREATE TABLE blobstore_dynamic_partitioning ( + id int, + name string, + dept string) +PARTITIONED BY ( + pt string, + dt string, + hr string) +LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_dynamic_partitioning'; + +INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr) SELECT id, name, dept, dt, hr FROM empty; + +SELECT COUNT(*) FROM blobstore_dynamic_partitioning; + +CREATE TABLE blobstore_list_bucketing ( + id int, + name string, + dept string) +PARTITIONED BY ( + pt string, + dt string, + hr string) +SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_list_bucketing'; + +INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty; + +SELECT COUNT(*) FROM blobstore_list_bucketing; + +-- Now test empty inserts with blobstore optimizations turned off. This should give us same results. +SET hive.blobstore.optimizations.enabled=false; + +INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr) SELECT id, name, dept, dt, hr FROM empty; +SELECT COUNT(*) FROM blobstore_dynamic_partitioning; + +INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b') SELECT id, name, dept FROM empty; +SELECT COUNT(*) FROM blobstore_list_bucketing; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/orc_buckets.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/orc_buckets.q b/itests/hive-blobstore/src/test/queries/clientpositive/orc_buckets.q new file mode 100644 index 0000000..9571842 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/orc_buckets.q @@ -0,0 +1,31 @@ +-- Test simple interaction with partitioned bucketed table with orc format in blobstore + +SET hive.exec.dynamic.partition=true; +SET hive.exec.reducers.max=10; +SET hive.exec.dynamic.partition.mode=nonstrict; + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_buckets/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE orc_buckets; +CREATE TABLE orc_buckets (a STRING, value DOUBLE) +PARTITIONED BY (b STRING) +CLUSTERED BY (a) INTO 10 BUCKETS +STORED AS ORC +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_buckets/orc_buckets'; + +INSERT OVERWRITE TABLE orc_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source; +SELECT * FROM orc_buckets; + +INSERT INTO TABLE orc_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source; +SELECT * FROM orc_buckets; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/orc_format_nonpart.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/orc_format_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/orc_format_nonpart.q new file mode 100644 index 0000000..ad95459 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/orc_format_nonpart.q @@ -0,0 +1,30 @@ +-- Test INSERT OVERWRITE and INSERT INTO on orc table in blobstore + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_format_nonpart/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE orc_table; +CREATE EXTERNAL TABLE orc_table (a INT, b STRING, value DOUBLE) STORED AS ORC +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_format_nonpart/orc_table'; + +INSERT OVERWRITE TABLE orc_table +SELECT * FROM blobstore_source; + +SELECT * FROM orc_table; +SELECT a FROM orc_table GROUP BY a; +SELECT b FROM orc_table GROUP BY b; +SELECT value FROM orc_table GROUP BY value; + +INSERT INTO TABLE orc_table +SELECT * FROM blobstore_source; + +SELECT * FROM orc_table; +SELECT a FROM orc_table GROUP BY a; +SELECT b FROM orc_table GROUP BY b; +SELECT value FROM orc_table GROUP BY value; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/orc_format_part.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/orc_format_part.q b/itests/hive-blobstore/src/test/queries/clientpositive/orc_format_part.q new file mode 100644 index 0000000..358eccd --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/orc_format_part.q @@ -0,0 +1,67 @@ +-- Test INSERT INTO and INSERT OVERWRITE on partitioned orc table in blobstore + +DROP TABLE src_events; +CREATE TABLE src_events +( + log_id BIGINT, + time BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_format_part/src_events/'; +LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events; + +DROP TABLE orc_events; +CREATE TABLE orc_events +( + log_id BIGINT, + time BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS ORC +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_format_part/orc_events'; + +SET hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; +SELECT COUNT(*) FROM orc_events WHERE run_date=20120921; +SELECT COUNT(*) FROM orc_events WHERE run_date=20121121; + +INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201211, game_id, event_name) +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; + +INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name) +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39; +SELECT COUNT(*) FROM orc_events; + +INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'; +SELECT COUNT(*) FROM orc_events; + +INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'; +SELECT COUNT(*) FROM orc_events; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/orc_nonstd_partitions_loc.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/orc_nonstd_partitions_loc.q b/itests/hive-blobstore/src/test/queries/clientpositive/orc_nonstd_partitions_loc.q new file mode 100644 index 0000000..c462538 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/orc_nonstd_partitions_loc.q @@ -0,0 +1,100 @@ +-- Test table in orc format with non-standard partition locations in blobstore + +DROP TABLE src_events; +CREATE TABLE src_events +( + log_id BIGINT, + time BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_nonstd_partitions_loc/src_events/'; +LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events; + +DROP TABLE orc_events; +CREATE TABLE orc_events +( + log_id BIGINT, + time BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS ORC +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_nonstd_partitions_loc/orc_events/'; + +SET hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; + +-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location +ALTER TABLE orc_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_nonstd_partitions_loc/orc_nonstd_loc/ns-part-1/'; +INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; +INSERT INTO TABLE orc_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; + +SET hive.merge.mapfiles=false; + +-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location with hive.merge.mapfiles false +ALTER TABLE orc_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_nonstd_partitions_loc/orc_nonstd_loc/ns-part-2/'; +INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; + +-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles false +INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; + +SET hive.merge.mapfiles=true; + +ALTER TABLE orc_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/orc_nonstd_partitions_loc/orc_nonstd_loc/ns-part-3/'; +INSERT INTO TABLE orc_events PARTITION (run_date=201207, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; + +-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles true +INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; + +ALTER TABLE orc_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change'); +ALTER TABLE orc_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change'); +ALTER TABLE orc_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change'); +SHOW PARTITIONS orc_events; +SELECT COUNT(*) FROM orc_events; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_buckets.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_buckets.q b/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_buckets.q new file mode 100644 index 0000000..606ef720 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_buckets.q @@ -0,0 +1,31 @@ +-- Test simple interaction with partitioned bucketed table with rcfile format in blobstore + +SET hive.exec.dynamic.partition=true; +SET hive.exec.reducers.max=10; +SET hive.exec.dynamic.partition.mode=nonstrict; + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_buckets/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE rcfile_buckets; +CREATE TABLE rcfile_buckets (a STRING, value DOUBLE) +PARTITIONED BY (b STRING) +CLUSTERED BY (a) INTO 10 BUCKETS +STORED AS RCFILE +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_buckets/rcfile_buckets'; + +INSERT OVERWRITE TABLE rcfile_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source; +SELECT * FROM rcfile_buckets; + +INSERT INTO TABLE rcfile_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source; +SELECT * FROM rcfile_buckets; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_format_nonpart.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_format_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_format_nonpart.q new file mode 100644 index 0000000..9cd909e --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_format_nonpart.q @@ -0,0 +1,30 @@ +-- Test INSERT OVERWRITE and INSERT INTO on rcfile table in blobstore + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_format_nonpart/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE rcfile_table; +CREATE TABLE rcfile_table (a INT, b STRING, value DOUBLE) STORED AS RCFILE +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_format_nonpart/rcfile_table'; + +INSERT OVERWRITE TABLE rcfile_table +SELECT * FROM blobstore_source; + +SELECT * FROM rcfile_table; +SELECT a FROM rcfile_table GROUP BY a; +SELECT b FROM rcfile_table GROUP BY b; +SELECT VALUE FROM rcfile_table GROUP BY VALUE; + +INSERT INTO TABLE rcfile_table +SELECT * FROM blobstore_source; + +SELECT * FROM rcfile_table; +SELECT a FROM rcfile_table GROUP BY a; +SELECT b FROM rcfile_table GROUP BY b; +SELECT value FROM rcfile_table GROUP BY value; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_format_part.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_format_part.q b/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_format_part.q new file mode 100644 index 0000000..c563d3a --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_format_part.q @@ -0,0 +1,67 @@ +-- Test INSERT INTO and INSERT OVERWRITE on partitioned rcfile table in blobstore + +DROP TABLE src_events; +CREATE TABLE src_events +( + log_id BIGINT, + time BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_format_part/src_events/'; +LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events; + +DROP TABLE rcfile_events; +CREATE TABLE rcfile_events +( + log_id BIGINT, + time BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS RCFILE +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_format_part/rcfile_events'; + +SET hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; +SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921; +SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121; + +INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201211, game_id, event_name) +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; + +INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name) +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39; +SELECT COUNT(*) FROM rcfile_events; + +INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'; +SELECT COUNT(*) FROM rcfile_events; + +INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'; +SELECT COUNT(*) FROM rcfile_events; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_nonstd_partitions_loc.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_nonstd_partitions_loc.q b/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_nonstd_partitions_loc.q new file mode 100644 index 0000000..d17c281 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/rcfile_nonstd_partitions_loc.q @@ -0,0 +1,100 @@ +-- Test table in rcfile format with non-standard partition locations in blobstore + +DROP TABLE src_events; +CREATE TABLE src_events +( + log_id BIGINT, + time BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_nonstd_partitions_loc/src_events/'; +LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events; + +DROP TABLE rcfile_events; +CREATE TABLE rcfile_events +( + log_id BIGINT, + time BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS RCFILE +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_nonstd_partitions_loc/rcfile_events/'; + +SET hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; + +-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location +ALTER TABLE rcfile_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_nonstd_partitions_loc/rcfile_nonstd_loc/ns-part-1/'; +INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; +INSERT INTO TABLE rcfile_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; + +SET hive.merge.mapfiles=false; + +-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location with hive.merge.mapfiles false +ALTER TABLE rcfile_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_nonstd_partitions_loc/rcfile_nonstd_loc/ns-part-2/'; +INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; + +-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles false +INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; + +SET hive.merge.mapfiles=true; + +ALTER TABLE rcfile_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/rcfile_nonstd_partitions_loc/rcfile_nonstd_loc/ns-part-3/'; +INSERT INTO TABLE rcfile_events PARTITION(run_date=201207,game_id=39, event_name='hq_change') +SELECT log_id,time,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; + +-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles true +INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; + +ALTER TABLE rcfile_events DROP PARTITION (run_date=201211,game_id=39, event_name='hq_change'); +ALTER TABLE rcfile_events DROP PARTITION (run_date=201209,game_id=39, event_name='hq_change'); +ALTER TABLE rcfile_events DROP PARTITION (run_date=201207,game_id=39, event_name='hq_change'); +SHOW PARTITIONS rcfile_events; +SELECT COUNT(*) FROM rcfile_events; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/zero_rows_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/zero_rows_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/zero_rows_blobstore.q new file mode 100644 index 0000000..1f663ef --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/zero_rows_blobstore.q @@ -0,0 +1,19 @@ +-- Insert overwrite into blobstore when WHERE clause returns zero rows +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source ( + key int +) +LOCATION '${hiveconf:test.blobstore.path.unique}/zero_rows_blobstore/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/kv6.txt' INTO TABLE blobstore_source; + +DROP TABLE blobstore_target; +CREATE TABLE blobstore_target ( + key int +) +LOCATION '${hiveconf:test.blobstore.path.unique}/zero_rows_blobstore/blobstore_target'; + +SELECT COUNT(*) FROM blobstore_target; +INSERT OVERWRITE TABLE blobstore_target SELECT key FROM blobstore_source; +SELECT COUNT(*) FROM blobstore_target; +INSERT OVERWRITE TABLE blobstore_target SELECT key FROM blobstore_source WHERE FALSE; +SELECT COUNT(*) FROM blobstore_target; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/queries/clientpositive/zero_rows_hdfs.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/zero_rows_hdfs.q b/itests/hive-blobstore/src/test/queries/clientpositive/zero_rows_hdfs.q new file mode 100644 index 0000000..ef3b71d --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/zero_rows_hdfs.q @@ -0,0 +1,18 @@ +-- Insert overwrite into hdfs from blobstore when WHERE clause returns zero rows +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source ( + key int +) +LOCATION '${hiveconf:test.blobstore.path.unique}/zero_rows_hdfs/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/kv6.txt' INTO TABLE blobstore_source; + +DROP TABLE hdfs_target; +CREATE TABLE hdfs_target ( + key int +); + +SELECT COUNT(*) FROM hdfs_target; +INSERT OVERWRITE TABLE hdfs_target SELECT key FROM blobstore_source; +SELECT COUNT(*) FROM hdfs_target; +INSERT OVERWRITE TABLE hdfs_target SELECT key FROM blobstore_source WHERE FALSE; +SELECT COUNT(*) FROM hdfs_target; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/resources/hive-site.xml b/itests/hive-blobstore/src/test/resources/hive-site.xml index 406b3b8..038db0d 100644 --- a/itests/hive-blobstore/src/test/resources/hive-site.xml +++ b/itests/hive-blobstore/src/test/resources/hive-site.xml @@ -279,6 +279,11 @@ <value>pblob:${system:test.tmp.dir}/bucket</value> </property> + <property> + <name>hive.exim.uri.scheme.whitelist</name> + <value>hdfs,pfile,file,s3,s3a,pblob</value> + </property> + <!-- To run these tests: # Create a file blobstore-conf.xml - DO NOT ADD TO REVISION CONTROL http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-blobstore/src/test/results/clientpositive/import_addpartition_blobstore_to_blobstore.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/import_addpartition_blobstore_to_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/import_addpartition_blobstore_to_blobstore.q.out new file mode 100644 index 0000000..c1e57ee --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/import_addpartition_blobstore_to_blobstore.q.out @@ -0,0 +1,283 @@ +PREHOOK: query: DROP TABLE exim_employee +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE exim_employee +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@exim_employee +POSTHOOK: query: CREATE TABLE exim_employee (emp_id int COMMENT "employee id") +COMMENT "employee table" +PARTITIONED BY (emp_country string COMMENT "two char iso code") +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@exim_employee +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@exim_employee +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="in") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@exim_employee +POSTHOOK: Output: default@exim_employee@emp_country=in +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@exim_employee +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="us") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@exim_employee +POSTHOOK: Output: default@exim_employee@emp_country=us +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@exim_employee +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/test.dat" +INTO TABLE exim_employee PARTITION (emp_country="cz") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@exim_employee +POSTHOOK: Output: default@exim_employee@emp_country=cz +PREHOOK: query: DESCRIBE EXTENDED exim_employee +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@exim_employee +POSTHOOK: query: DESCRIBE EXTENDED exim_employee +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@exim_employee +emp_id int employee id +emp_country string two char iso code + +# Partition Information +# col_name data_type comment + +emp_country string two char iso code + +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM exim_employee +PREHOOK: type: QUERY +PREHOOK: Input: default@exim_employee +PREHOOK: Input: default@exim_employee@emp_country=cz +PREHOOK: Input: default@exim_employee@emp_country=in +PREHOOK: Input: default@exim_employee@emp_country=us +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM exim_employee +POSTHOOK: type: QUERY +POSTHOOK: Input: default@exim_employee +POSTHOOK: Input: default@exim_employee@emp_country=cz +POSTHOOK: Input: default@exim_employee@emp_country=in +POSTHOOK: Input: default@exim_employee@emp_country=us +#### A masked pattern was here #### +1 cz +2 cz +3 cz +4 cz +5 cz +6 cz +1 in +2 in +3 in +4 in +5 in +6 in +1 us +2 us +3 us +4 us +5 us +6 us +PREHOOK: query: EXPORT TABLE exim_employee +TO '### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee' +PREHOOK: type: EXPORT +PREHOOK: Input: default@exim_employee@emp_country=cz +PREHOOK: Input: default@exim_employee@emp_country=in +PREHOOK: Input: default@exim_employee@emp_country=us +PREHOOK: Output: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee +POSTHOOK: query: EXPORT TABLE exim_employee +TO '### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee' +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@exim_employee@emp_country=cz +POSTHOOK: Input: default@exim_employee@emp_country=in +POSTHOOK: Input: default@exim_employee@emp_country=us +POSTHOOK: Output: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee +PREHOOK: query: DROP TABLE exim_employee +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@exim_employee +PREHOOK: Output: default@exim_employee +POSTHOOK: query: DROP TABLE exim_employee +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@exim_employee +POSTHOOK: Output: default@exim_employee +PREHOOK: query: IMPORT TABLE exim_employee PARTITION (emp_country='us') +FROM '### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee' +#### A masked pattern was here #### +PREHOOK: type: IMPORT +PREHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee +PREHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/import/exim_employee +PREHOOK: Output: database:default +POSTHOOK: query: IMPORT TABLE exim_employee PARTITION (emp_country='us') +FROM '### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee' +#### A masked pattern was here #### +POSTHOOK: type: IMPORT +POSTHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee +POSTHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/import/exim_employee +POSTHOOK: Output: database:default +POSTHOOK: Output: default@exim_employee +POSTHOOK: Output: default@exim_employee@emp_country=us +PREHOOK: query: DESCRIBE EXTENDED exim_employee +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@exim_employee +POSTHOOK: query: DESCRIBE EXTENDED exim_employee +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@exim_employee +emp_id int employee id +emp_country string two char iso code + +# Partition Information +# col_name data_type comment + +emp_country string two char iso code + +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM exim_employee +PREHOOK: type: QUERY +PREHOOK: Input: default@exim_employee +PREHOOK: Input: default@exim_employee@emp_country=us +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM exim_employee +POSTHOOK: type: QUERY +POSTHOOK: Input: default@exim_employee +POSTHOOK: Input: default@exim_employee@emp_country=us +#### A masked pattern was here #### +1 us +2 us +3 us +4 us +5 us +6 us +PREHOOK: query: IMPORT TABLE exim_employee PARTITION (emp_country='cz') +FROM '### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee' +#### A masked pattern was here #### +PREHOOK: type: IMPORT +PREHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee +PREHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/import/exim_employee +PREHOOK: Output: default@exim_employee +POSTHOOK: query: IMPORT TABLE exim_employee PARTITION (emp_country='cz') +FROM '### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee' +#### A masked pattern was here #### +POSTHOOK: type: IMPORT +POSTHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee +POSTHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/import/exim_employee +POSTHOOK: Output: default@exim_employee +POSTHOOK: Output: default@exim_employee@emp_country=cz +PREHOOK: query: DESCRIBE EXTENDED exim_employee +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@exim_employee +POSTHOOK: query: DESCRIBE EXTENDED exim_employee +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@exim_employee +emp_id int employee id +emp_country string two char iso code + +# Partition Information +# col_name data_type comment + +emp_country string two char iso code + +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM exim_employee +PREHOOK: type: QUERY +PREHOOK: Input: default@exim_employee +PREHOOK: Input: default@exim_employee@emp_country=cz +PREHOOK: Input: default@exim_employee@emp_country=us +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM exim_employee +POSTHOOK: type: QUERY +POSTHOOK: Input: default@exim_employee +POSTHOOK: Input: default@exim_employee@emp_country=cz +POSTHOOK: Input: default@exim_employee@emp_country=us +#### A masked pattern was here #### +1 cz +2 cz +3 cz +4 cz +5 cz +6 cz +1 us +2 us +3 us +4 us +5 us +6 us +PREHOOK: query: IMPORT TABLE exim_employee PARTITION (emp_country='in') +FROM '### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee' +#### A masked pattern was here #### +PREHOOK: type: IMPORT +PREHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee +PREHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/import/exim_employee +PREHOOK: Output: default@exim_employee +POSTHOOK: query: IMPORT TABLE exim_employee PARTITION (emp_country='in') +FROM '### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee' +#### A masked pattern was here #### +POSTHOOK: type: IMPORT +POSTHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/export/exim_employee +POSTHOOK: Input: ### test.blobstore.path ###/import_addpartition_blobstore_to_blobstore/import/exim_employee +POSTHOOK: Output: default@exim_employee +POSTHOOK: Output: default@exim_employee@emp_country=in +PREHOOK: query: DESCRIBE EXTENDED exim_employee +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@exim_employee +POSTHOOK: query: DESCRIBE EXTENDED exim_employee +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@exim_employee +emp_id int employee id +emp_country string two char iso code + +# Partition Information +# col_name data_type comment + +emp_country string two char iso code + +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM exim_employee +PREHOOK: type: QUERY +PREHOOK: Input: default@exim_employee +PREHOOK: Input: default@exim_employee@emp_country=cz +PREHOOK: Input: default@exim_employee@emp_country=in +PREHOOK: Input: default@exim_employee@emp_country=us +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM exim_employee +POSTHOOK: type: QUERY +POSTHOOK: Input: default@exim_employee +POSTHOOK: Input: default@exim_employee@emp_country=cz +POSTHOOK: Input: default@exim_employee@emp_country=in +POSTHOOK: Input: default@exim_employee@emp_country=us +#### A masked pattern was here #### +1 cz +2 cz +3 cz +4 cz +5 cz +6 cz +1 in +2 in +3 in +4 in +5 in +6 in +1 us +2 us +3 us +4 us +5 us +6 us