hive hook fixes for hdp 2.2.4
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/64c78442 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/64c78442 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/64c78442 Branch: refs/remotes/origin/master Commit: 64c784428b7c87aab5e6a8311c9121de48e9183e Parents: 4b2a4ca Author: Shwetha GS <[email protected]> Authored: Mon May 4 18:04:38 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Mon May 4 18:04:38 2015 +0530 ---------------------------------------------------------------------- addons/hive-bridge/pom.xml | 8 +--- .../hive/bridge/HiveMetaStoreBridge.java | 41 +++++++++++--------- .../hadoop/metadata/hive/hook/HiveHook.java | 5 ++- .../hadoop/metadata/hive/hook/HiveHookIT.java | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64c78442/addons/hive-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index b908847..03d2dcf 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -90,15 +90,9 @@ <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>${hadoop.version}</version> + <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64c78442/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java index ab14744..6084a68 100755 --- a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java @@ -19,15 +19,15 @@ package org.apache.hadoop.metadata.hive.bridge; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.metadata.MetadataServiceClient; import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator; import org.apache.hadoop.metadata.hive.model.HiveDataTypes; @@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** * A Bridge Utility that imports metadata from the Hive Meta Store @@ -66,7 +67,7 @@ public class HiveMetaStoreBridge { private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class); - private final HiveMetaStoreClient hiveMetaStoreClient; + private final Hive hiveClient; private final MetadataServiceClient metadataServiceClient; /** @@ -74,7 +75,7 @@ public class HiveMetaStoreBridge { * @param hiveConf */ public HiveMetaStoreBridge(HiveConf hiveConf) throws Exception { - hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf); + hiveClient = Hive.get(hiveConf); metadataServiceClient = new MetadataServiceClient(hiveConf.get(DGI_URL_PROPERTY, DEFAULT_DGI_URL)); } @@ -88,7 +89,7 @@ public class HiveMetaStoreBridge { } private void importDatabases() throws Exception { - List<String> databases = hiveMetaStoreClient.getAllDatabases(); + List<String> databases = hiveClient.getAllDatabases(); for (String databaseName : databases) { Referenceable dbReference = registerDatabase(databaseName); @@ -99,7 +100,7 @@ public class HiveMetaStoreBridge { public Referenceable registerDatabase(String databaseName) throws Exception { LOG.info("Importing objects from databaseName : " + databaseName); - Database hiveDB = hiveMetaStoreClient.getDatabase(databaseName); + Database hiveDB = hiveClient.getDatabase(databaseName); Referenceable dbRef = new Referenceable(HiveDataTypes.HIVE_DB.getName()); dbRef.set("name", hiveDB.getName()); @@ -128,7 +129,7 @@ public class HiveMetaStoreBridge { } private void importTables(String databaseName, Referenceable databaseReferenceable) throws Exception { - List<String> hiveTables = hiveMetaStoreClient.getAllTables(databaseName); + List<String> hiveTables = hiveClient.getAllTables(databaseName); for (String tableName : hiveTables) { Pair<Referenceable, Referenceable> tableReferenceable = registerTable(databaseReferenceable, databaseName, tableName); @@ -144,12 +145,13 @@ public class HiveMetaStoreBridge { public Pair<Referenceable, Referenceable> registerTable(Referenceable dbReference, String dbName, String tableName) throws Exception { LOG.info("Importing objects from " + dbName + "." + tableName); - Table hiveTable = hiveMetaStoreClient.getTable(dbName, tableName); + Table hiveTable = hiveClient.getTable(dbName, tableName); Referenceable tableRef = new Referenceable(HiveDataTypes.HIVE_TABLE.getName()); tableRef.set("tableName", hiveTable.getTableName()); tableRef.set("owner", hiveTable.getOwner()); - tableRef.set("createTime", hiveTable.getCreateTime()); + //todo fix + tableRef.set("createTime", hiveTable.getLastAccessTime()); tableRef.set("lastAccessTime", hiveTable.getLastAccessTime()); tableRef.set("retention", hiveTable.getRetention()); @@ -164,7 +166,7 @@ public class HiveMetaStoreBridge { // add reference to the Partition Keys List<Referenceable> partKeys = new ArrayList<>(); Referenceable colRef; - if (hiveTable.getPartitionKeysSize() > 0) { + if (hiveTable.getPartitionKeys().size() > 0) { for (FieldSchema fs : hiveTable.getPartitionKeys()) { colRef = new Referenceable(HiveDataTypes.HIVE_COLUMN.getName()); colRef.set("name", fs.getName()); @@ -179,11 +181,11 @@ public class HiveMetaStoreBridge { tableRef.set("parameters", hiveTable.getParameters()); - if (hiveTable.isSetViewOriginalText()) { + if (hiveTable.getViewOriginalText() != null) { tableRef.set("viewOriginalText", hiveTable.getViewOriginalText()); } - if (hiveTable.isSetViewExpandedText()) { + if (hiveTable.getViewExpandedText() != null) { tableRef.set("viewExpandedText", hiveTable.getViewExpandedText()); } @@ -197,12 +199,14 @@ public class HiveMetaStoreBridge { return Pair.of(tableReferenceable, sdReferenceable); } - private void importPartitions(String db, String table, + private void importPartitions(String db, String tableName, Referenceable dbReferenceable, Referenceable tableReferenceable, Referenceable sdReferenceable) throws Exception { - List<Partition> tableParts = hiveMetaStoreClient.listPartitions( - db, table, Short.MAX_VALUE); + Table table = new Table(); + table.setDbName(db); + table.setTableName(tableName); + Set<Partition> tableParts = hiveClient.getAllPartitionsOf(table); if (tableParts.size() > 0) { for (Partition hivePart : tableParts) { @@ -221,7 +225,8 @@ public class HiveMetaStoreBridge { partRef.set("dbName", dbReferenceable); partRef.set("tableName", tableReferenceable); - partRef.set("createTime", hivePart.getCreateTime()); + //todo fix + partRef.set("createTime", hivePart.getLastAccessTime()); partRef.set("lastAccessTime", hivePart.getLastAccessTime()); // sdStruct = fillStorageDescStruct(hivePart.getSd()); @@ -237,7 +242,7 @@ public class HiveMetaStoreBridge { private void importIndexes(String db, String table, Referenceable dbReferenceable, Referenceable tableReferenceable) throws Exception { - List<Index> indexes = hiveMetaStoreClient.listIndexes(db, table, Short.MAX_VALUE); + List<Index> indexes = hiveClient.getIndexes(db, table, Short.MAX_VALUE); if (indexes.size() > 0) { for (Index index : indexes) { importIndex(index, dbReferenceable, tableReferenceable); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64c78442/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java index abffe2e..4af7178 100755 --- a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java @@ -155,7 +155,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo // clone to avoid concurrent access final HiveConf conf = new HiveConf(hookContext.getConf()); - boolean debug = conf.get("debug", "false").equals("true"); + boolean debug = conf.get("hive.hook.dgi.synchronous", "false").equals("true"); if (debug) { fireAndForget(hookContext, conf); @@ -178,8 +178,9 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo private void fireAndForget(HookContext hookContext, HiveConf conf) throws Exception { assert hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK : "Non-POST_EXEC_HOOK not supported!"; + LOG.info("Entered DGI hook for hook type {} operation {}", hookContext.getHookType(), + hookContext.getOperationName()); HiveOperation operation = HiveOperation.valueOf(hookContext.getOperationName()); - LOG.info("Entered DGI hook for hook type {} operation {}", hookContext.getHookType(), operation); HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(conf); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64c78442/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java index c47bdd5..7b6ba1b 100755 --- a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java @@ -58,7 +58,7 @@ public class HiveHookIT { hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/metastore"); hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL); hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true"); - hiveConf.set("debug", "true"); + hiveConf.set("hive.hook.dgi.synchronous", "true"); return hiveConf; }
