Repository: hive Updated Branches: refs/heads/branch-3 a2c08792a -> 90b442c1b
Revert "Merge branch 'branch-3' of http://git-wip-us.apache.org/repos/asf/hive into branch-3" This reverts commit 9d80c2d6a1a7bf378d2c6a28006d09f3c3643e5f, reversing changes made to 6f9a76744ea3acb015a72e8d1e94a335ac80f42f. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/90b442c1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/90b442c1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/90b442c1 Branch: refs/heads/branch-3 Commit: 90b442c1b54f786558e04cec60715756e7accd1d Parents: a2c0879 Author: Deepak Jaiswal <djais...@apache.org> Authored: Mon Jun 18 10:57:53 2018 -0700 Committer: Deepak Jaiswal <djais...@apache.org> Committed: Mon Jun 18 10:57:53 2018 -0700 ---------------------------------------------------------------------- .../TestTransactionalValidationListener.java | 127 ------------------- .../TransactionalValidationListener.java | 23 +--- .../metastore/client/MetaStoreClientTest.java | 2 +- 3 files changed, 6 insertions(+), 146 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/90b442c1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java deleted file mode 100644 index 3aaad22..0000000 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.apache.hadoop.hive.metastore; - -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; -import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.client.MetaStoreClientTest; -import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; -import org.apache.hadoop.hive.ql.io.AcidUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@RunWith(Parameterized.class) -public class TestTransactionalValidationListener extends MetaStoreClientTest { - - private AbstractMetaStoreService metaStore; - private IMetaStoreClient client; - private boolean createdCatalogs = false; - - @BeforeClass - public static void startMetaStores() { - Map<MetastoreConf.ConfVars, String> msConf = new HashMap<MetastoreConf.ConfVars, String>(); - - // Enable TransactionalValidationListener + create.as.acid - Map<String, String> extraConf = new HashMap<>(); - extraConf.put("metastore.create.as.acid", "true"); - extraConf.put("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); - extraConf.put("hive.support.concurrency", "true"); - startMetaStores(msConf, extraConf); - } - - @Before - public void setUp() throws Exception { - // Get new client - client = metaStore.getClient(); - if (!createdCatalogs) { - createCatalogs(); - createdCatalogs = true; - } - } - - @After - public void tearDown() throws Exception { - try { - if (client != null) { - client.close(); - } - } finally { - client = null; - } - } - - public TestTransactionalValidationListener(String name, AbstractMetaStoreService metaStore) throws Exception { - this.metaStore = metaStore; - } - - private void createCatalogs() throws Exception { - String[] catNames = {"spark", "myapp"}; - String[] location = {MetaStoreTestUtils.getTestWarehouseDir("spark"), - MetaStoreTestUtils.getTestWarehouseDir("myapp")}; - - for (int i = 0; i < catNames.length; i++) { - Catalog cat = new CatalogBuilder() - .setName(catNames[i]) - .setLocation(location[i]) - .build(); - client.createCatalog(cat); - File dir = new File(cat.getLocationUri()); - Assert.assertTrue(dir.exists() && dir.isDirectory()); - } - } - - private Table createOrcTable(String catalog) throws Exception { - Table table = new Table(); - StorageDescriptor sd = new StorageDescriptor(); - List<FieldSchema> cols = new ArrayList<>(); - - table.setDbName("default"); - table.setTableName("test_table"); - cols.add(new FieldSchema("column_name", "int", null)); - sd.setCols(cols); - sd.setSerdeInfo(new SerDeInfo()); - sd.setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"); - sd.setOutputFormat("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"); - table.setSd(sd); - table.setCatName(catalog); - table.setTableType("MANAGED_TABLE"); - - client.createTable(table); - Table createdTable = client.getTable(catalog, table.getDbName(), table.getTableName()); - return createdTable; - } - - @Test - public void testCreateAsAcid() throws Exception { - // Table created in hive catalog should have been automatically set to transactional - Table createdTable = createOrcTable("hive"); - assertTrue(AcidUtils.isTransactionalTable(createdTable)); - - // Non-hive catalogs should not be transactional - createdTable = createOrcTable("spark"); - assertFalse(AcidUtils.isTransactionalTable(createdTable)); - - createdTable = createOrcTable("myapp"); - assertFalse(AcidUtils.isTransactionalTable(createdTable)); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/90b442c1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 33cf542..76069bb 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -53,11 +53,8 @@ public final class TransactionalValidationListener extends MetaStorePreEventList public static final String DEFAULT_TRANSACTIONAL_PROPERTY = "default"; public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only"; - private final Set<String> supportedCatalogs = new HashSet<String>(); - TransactionalValidationListener(Configuration conf) { super(conf); - supportedCatalogs.add("hive"); } @Override @@ -76,21 +73,11 @@ public final class TransactionalValidationListener extends MetaStorePreEventList } private void handle(PreAlterTableEvent context) throws MetaException { - if (supportedCatalogs.contains(getTableCatalog(context.getNewTable()))) { - handleAlterTableTransactionalProp(context); - } + handleAlterTableTransactionalProp(context); } private void handle(PreCreateTableEvent context) throws MetaException { - if (supportedCatalogs.contains(getTableCatalog(context.getTable()))) { - handleCreateTableTransactionalProp(context); - } - } - - private String getTableCatalog(Table table) { - String catName = table.isSetCatName() ? table.getCatName() : - MetaStoreUtils.getDefaultCatalog(getConf()); - return catName.toLowerCase(); + handleCreateTableTransactionalProp(context); } /** @@ -243,8 +230,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList newTable.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)); return; } - - Configuration conf = getConf(); + Configuration conf = MetastoreConf.newMetastoreConf(); boolean makeAcid = //no point making an acid table if these other props are not set since it will just throw //exceptions when someone tries to use the table. @@ -451,7 +437,8 @@ public final class TransactionalValidationListener extends MetaStorePreEventList try { Warehouse wh = hmsHandler.getWh(); if (table.getSd().getLocation() == null || table.getSd().getLocation().isEmpty()) { - String catName = getTableCatalog(table); + String catName = table.isSetCatName() ? table.getCatName() : + MetaStoreUtils.getDefaultCatalog(getConf()); tablePath = wh.getDefaultTablePath(hmsHandler.getMS().getDatabase( catName, table.getDbName()), table); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/90b442c1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java index dc48fa8..a0e9d32 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java @@ -67,7 +67,7 @@ public abstract class MetaStoreClientTest { * @param msConf Specific MetaStore configuration values * @param extraConf Specific other configuration values */ - public static void startMetaStores(Map<MetastoreConf.ConfVars, String> msConf, + static void startMetaStores(Map<MetastoreConf.ConfVars, String> msConf, Map<String, String> extraConf) { for(AbstractMetaStoreService metaStoreService : metaStoreServices) { try {
