HIVE-19898: Disable TransactionalValidationListener when the table is not in the Hive catalog (Jason Dere, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ebd2c5f8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ebd2c5f8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ebd2c5f8 Branch: refs/heads/master-txnstats Commit: ebd2c5f8a82b35eabca146520ffcd87605084618 Parents: 766c3dc Author: Jason Dere <jd...@hortonworks.com> Authored: Sun Jun 17 21:53:23 2018 -0700 Committer: Jason Dere <jd...@hortonworks.com> Committed: Sun Jun 17 21:53:23 2018 -0700 ---------------------------------------------------------------------- .../TestTransactionalValidationListener.java | 127 +++++++++++++++++++ .../TransactionalValidationListener.java | 23 +++- .../metastore/client/MetaStoreClientTest.java | 2 +- 3 files changed, 146 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ebd2c5f8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java new file mode 100644 index 0000000..3aaad22 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java @@ -0,0 +1,127 @@ +package org.apache.hadoop.hive.metastore; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.api.Catalog; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.MetaStoreClientTest; +import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(Parameterized.class) +public class TestTransactionalValidationListener extends MetaStoreClientTest { + + private AbstractMetaStoreService metaStore; + private IMetaStoreClient client; + private boolean createdCatalogs = false; + + @BeforeClass + public static void startMetaStores() { + Map<MetastoreConf.ConfVars, String> msConf = new HashMap<MetastoreConf.ConfVars, String>(); + + // Enable TransactionalValidationListener + create.as.acid + Map<String, String> extraConf = new HashMap<>(); + extraConf.put("metastore.create.as.acid", "true"); + extraConf.put("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); + extraConf.put("hive.support.concurrency", "true"); + startMetaStores(msConf, extraConf); + } + + @Before + public void setUp() throws Exception { + // Get new client + client = metaStore.getClient(); + if (!createdCatalogs) { + createCatalogs(); + createdCatalogs = true; + } + } + + @After + public void tearDown() throws Exception { + try { + if (client != null) { + client.close(); + } + } finally { + client = null; + } + } + + public TestTransactionalValidationListener(String name, AbstractMetaStoreService metaStore) throws Exception { + this.metaStore = metaStore; + } + + private void createCatalogs() throws Exception { + String[] catNames = {"spark", "myapp"}; + String[] location = {MetaStoreTestUtils.getTestWarehouseDir("spark"), + MetaStoreTestUtils.getTestWarehouseDir("myapp")}; + + for (int i = 0; i < catNames.length; i++) { + Catalog cat = new CatalogBuilder() + .setName(catNames[i]) + .setLocation(location[i]) + .build(); + client.createCatalog(cat); + File dir = new File(cat.getLocationUri()); + Assert.assertTrue(dir.exists() && dir.isDirectory()); + } + } + + private Table createOrcTable(String catalog) throws Exception { + Table table = new Table(); + StorageDescriptor sd = new StorageDescriptor(); + List<FieldSchema> cols = new ArrayList<>(); + + table.setDbName("default"); + table.setTableName("test_table"); + cols.add(new FieldSchema("column_name", "int", null)); + sd.setCols(cols); + sd.setSerdeInfo(new SerDeInfo()); + sd.setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"); + sd.setOutputFormat("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"); + table.setSd(sd); + table.setCatName(catalog); + table.setTableType("MANAGED_TABLE"); + + client.createTable(table); + Table createdTable = client.getTable(catalog, table.getDbName(), table.getTableName()); + return createdTable; + } + + @Test + public void testCreateAsAcid() throws Exception { + // Table created in hive catalog should have been automatically set to transactional + Table createdTable = createOrcTable("hive"); + assertTrue(AcidUtils.isTransactionalTable(createdTable)); + + // Non-hive catalogs should not be transactional + createdTable = createOrcTable("spark"); + assertFalse(AcidUtils.isTransactionalTable(createdTable)); + + createdTable = createOrcTable("myapp"); + assertFalse(AcidUtils.isTransactionalTable(createdTable)); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ebd2c5f8/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 76069bb..33cf542 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -53,8 +53,11 @@ public final class TransactionalValidationListener extends MetaStorePreEventList public static final String DEFAULT_TRANSACTIONAL_PROPERTY = "default"; public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only"; + private final Set<String> supportedCatalogs = new HashSet<String>(); + TransactionalValidationListener(Configuration conf) { super(conf); + supportedCatalogs.add("hive"); } @Override @@ -73,11 +76,21 @@ public final class TransactionalValidationListener extends MetaStorePreEventList } private void handle(PreAlterTableEvent context) throws MetaException { - handleAlterTableTransactionalProp(context); + if (supportedCatalogs.contains(getTableCatalog(context.getNewTable()))) { + handleAlterTableTransactionalProp(context); + } } private void handle(PreCreateTableEvent context) throws MetaException { - handleCreateTableTransactionalProp(context); + if (supportedCatalogs.contains(getTableCatalog(context.getTable()))) { + handleCreateTableTransactionalProp(context); + } + } + + private String getTableCatalog(Table table) { + String catName = table.isSetCatName() ? table.getCatName() : + MetaStoreUtils.getDefaultCatalog(getConf()); + return catName.toLowerCase(); } /** @@ -230,7 +243,8 @@ public final class TransactionalValidationListener extends MetaStorePreEventList newTable.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)); return; } - Configuration conf = MetastoreConf.newMetastoreConf(); + + Configuration conf = getConf(); boolean makeAcid = //no point making an acid table if these other props are not set since it will just throw //exceptions when someone tries to use the table. @@ -437,8 +451,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList try { Warehouse wh = hmsHandler.getWh(); if (table.getSd().getLocation() == null || table.getSd().getLocation().isEmpty()) { - String catName = table.isSetCatName() ? table.getCatName() : - MetaStoreUtils.getDefaultCatalog(getConf()); + String catName = getTableCatalog(table); tablePath = wh.getDefaultTablePath(hmsHandler.getMS().getDatabase( catName, table.getDbName()), table); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/ebd2c5f8/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java index a0e9d32..dc48fa8 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java @@ -67,7 +67,7 @@ public abstract class MetaStoreClientTest { * @param msConf Specific MetaStore configuration values * @param extraConf Specific other configuration values */ - static void startMetaStores(Map<MetastoreConf.ConfVars, String> msConf, + public static void startMetaStores(Map<MetastoreConf.ConfVars, String> msConf, Map<String, String> extraConf) { for(AbstractMetaStoreService metaStoreService : metaStoreServices) { try {