This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push: new 0323ce3 ATLAS-3033: Skip hive temp table while getting Table object details from hiveContext. 0323ce3 is described below commit 0323ce3825c63b06d7b0032edd10b73b84078a4a Author: nixonrodrigues <ni...@apache.org> AuthorDate: Thu May 2 20:37:08 2019 -0700 ATLAS-3033: Skip hive temp table while getting Table object details from hiveContext. Signed-off-by: Madhan Neethiraj <mad...@apache.org> (cherry picked from commit b5472ce313dcf24b0bffbe5201f0c95eec025848) --- .../atlas/hive/hook/AtlasHiveHookContext.java | 12 ++++++++++ .../java/org/apache/atlas/hive/hook/HiveHook.java | 26 ++++++++++++++++++++++ .../atlas/hive/hook/events/BaseHiveEvent.java | 20 +++++++++++++---- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java index 4b2b143..54cd48f 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java @@ -151,6 +151,18 @@ public class AtlasHiveHookContext { return hook.getPreprocessActionForHiveTable(qualifiedName); } + public List getIgnoreDummyDatabaseName() { + return hook.getIgnoreDummyDatabaseName(); + } + + public List getIgnoreDummyTableName() { + return hook.getIgnoreDummyTableName(); + } + + public String getIgnoreValuesTmpTableNamePrefix() { + return hook.getIgnoreValuesTmpTableNamePrefix(); + } + public String getQualifiedName(Database db) { return (db.getName() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); } diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index c94d6ef..c0094c6 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; import org.apache.hadoop.hive.ql.hooks.HookContext; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; @@ -83,6 +84,9 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { private static final List<Pattern> hiveTablesToIgnore = new ArrayList<>(); private static final List<Pattern> hiveTablesToPrune = new ArrayList<>(); private static final Map<String, PreprocessAction> hiveTablesCache; + private static final List ignoreDummyDatabaseName; + private static final List ignoreDummyTableName; + private static final String ignoreValuesTmpTableNamePrefix; private static HiveHookObjectNamesCache knownObjects = null; private static String hostName; @@ -138,6 +142,16 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { knownObjects = nameCacheEnabled ? new HiveHookObjectNamesCache(nameCacheDatabaseMaxCount, nameCacheTableMaxCount, nameCacheRebuildIntervalSeconds) : null; + List<String> defaultDummyDatabase = new ArrayList<>(); + List<String> defaultDummyTable = new ArrayList<>(); + + defaultDummyDatabase.add(SemanticAnalyzer.DUMMY_DATABASE); + defaultDummyTable.add(SemanticAnalyzer.DUMMY_TABLE); + + ignoreDummyDatabaseName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.database.name", defaultDummyDatabase); + ignoreDummyTableName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.table.name", defaultDummyTable); + ignoreValuesTmpTableNamePrefix = atlasProperties.getString("atlas.hook.hive.ignore.values.tmp.table.name.prefix", "Values__Tmp__Table__"); + try { hostName = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { @@ -256,6 +270,18 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { return skipHiveColumnLineageHive20633InputsThreshold; } + public List getIgnoreDummyDatabaseName() { + return ignoreDummyDatabaseName; + } + + public List getIgnoreDummyTableName() { + return ignoreDummyTableName; + } + + public String getIgnoreValuesTmpTableNamePrefix() { + return ignoreValuesTmpTableNamePrefix; + } + public PreprocessAction getPreprocessActionForHiveTable(String qualifiedName) { PreprocessAction ret = PreprocessAction.NONE; diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java index 4b9cdb9..f378d84 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java @@ -246,17 +246,29 @@ public abstract class BaseHiveEvent { switch (entity.getType()) { case DATABASE: { - Database db = getHive().getDatabase(entity.getDatabase().getName()); + if (!context.getIgnoreDummyDatabaseName().contains(entity.getDatabase().getName())) { + Database db = getHive().getDatabase(entity.getDatabase().getName()); - ret = toDbEntity(db); + ret = toDbEntity(db); + } } break; case TABLE: case PARTITION: { - Table table = getHive().getTable(entity.getTable().getDbName(), entity.getTable().getTableName()); + String dbName = entity.getTable().getDbName(); + String tableName = entity.getTable().getTableName(); + boolean skipTable = StringUtils.isNotEmpty(context.getIgnoreValuesTmpTableNamePrefix()) && tableName.toLowerCase().startsWith(context.getIgnoreValuesTmpTableNamePrefix()); + + if (!skipTable) { + skipTable = context.getIgnoreDummyTableName().contains(tableName) && context.getIgnoreDummyDatabaseName().contains(dbName); + } - ret = toTableEntity(table, entityExtInfo); + if (!skipTable) { + Table table = getHive().getTable(dbName, tableName); + + ret = toTableEntity(table, entityExtInfo); + } } break;