Repository: atlas Updated Branches: refs/heads/branch-0.8 4a72f2062 -> a37aeda48 (forced update)
ATLAS-2961: updated Hive hook to recognize configuration 'hdfs_path.convert_to_lowercase' Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a37aeda4 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a37aeda4 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a37aeda4 Branch: refs/heads/branch-0.8 Commit: a37aeda48aa4239cd110e665c837c730edb4b800 Parents: 8d7e94d Author: Madhan Neethiraj <mad...@apache.org> Authored: Fri Nov 9 09:00:13 2018 -0800 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Sat Nov 10 14:52:12 2018 -0800 ---------------------------------------------------------------------- .../atlas/hive/bridge/HiveMetaStoreBridge.java | 4 ++-- .../atlas/hive/hook/AtlasHiveHookContext.java | 4 ++++ .../org/apache/atlas/hive/hook/HiveHook.java | 7 +++++++ .../atlas/hive/hook/events/BaseHiveEvent.java | 22 +++++++++++++++----- 4 files changed, 30 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/a37aeda4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index d6a1b37..2f6d1fa 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -204,7 +204,7 @@ public class HiveMetaStoreBridge { * @param hiveConf {@link HiveConf} for Hive component in the cluster */ public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf, AtlasClientV2 atlasClientV2) throws Exception { - this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, true)); + this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false)); } /** @@ -690,7 +690,7 @@ public class HiveMetaStoreBridge { AtlasEntity ret = new AtlasEntity(HDFS_PATH); Path path = new Path(pathUri); - ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase()); + ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString()); ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, clusterName); ret.setAttribute(ATTRIBUTE_PATH, pathUri); http://git-wip-us.apache.org/repos/asf/atlas/blob/a37aeda4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java index b467f4c..23cb853 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java @@ -85,6 +85,10 @@ public class AtlasHiveHookContext { return hook.getClusterName(); } + public boolean isConvertHdfsPathToLowerCase() { + return hook.isConvertHdfsPathToLowerCase(); + } + public boolean getSkipHiveColumnLineageHive20633() { return hook.getSkipHiveColumnLineageHive20633(); } http://git-wip-us.apache.org/repos/asf/atlas/blob/a37aeda4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index 501eb63..7b60553 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -47,6 +47,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { public static final String CONF_PREFIX = "atlas.hook.hive."; public static final String CONF_CLUSTER_NAME = "atlas.cluster.name"; + public static final String HDFS_PATH_CONVERT_TO_LOWER_CASE = CONF_PREFIX + "hdfs_path.convert_to_lowercase"; public static final String HOOK_NAME_CACHE_ENABLED = CONF_PREFIX + "name.cache.enabled"; public static final String HOOK_NAME_CACHE_DATABASE_COUNT = CONF_PREFIX + "name.cache.database.count"; public static final String HOOK_NAME_CACHE_TABLE_COUNT = CONF_PREFIX + "name.cache.table.count"; @@ -59,6 +60,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { private static final Map<String, HiveOperation> OPERATION_MAP = new HashMap<>(); private static final String clusterName; + private static final boolean convertHdfsPathToLowerCase; private static final boolean nameCacheEnabled; private static final int nameCacheDatabaseMaxCount; private static final int nameCacheTableMaxCount; @@ -75,6 +77,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { } clusterName = atlasProperties.getString(CONF_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); + convertHdfsPathToLowerCase = atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false); nameCacheEnabled = atlasProperties.getBoolean(HOOK_NAME_CACHE_ENABLED, true); nameCacheDatabaseMaxCount = atlasProperties.getInt(HOOK_NAME_CACHE_DATABASE_COUNT, 10000); nameCacheTableMaxCount = atlasProperties.getInt(HOOK_NAME_CACHE_TABLE_COUNT, 10000); @@ -189,6 +192,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { return clusterName; } + public boolean isConvertHdfsPathToLowerCase() { + return convertHdfsPathToLowerCase; + } + public boolean getSkipHiveColumnLineageHive20633() { return skipHiveColumnLineageHive20633; } http://git-wip-us.apache.org/repos/asf/atlas/blob/a37aeda4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java index 9109a0f..2d90a15 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java @@ -124,6 +124,7 @@ public abstract class BaseHiveEvent { public static final String ATTRIBUTE_EXPRESSION = "expression"; public static final String ATTRIBUTE_ALIASES = "aliases"; + public static final String HDFS_PATH_PREFIX = "hdfs://"; public static final long MILLIS_CONVERT_FACTOR = 1000; @@ -481,7 +482,14 @@ public abstract class BaseHiveEvent { } protected AtlasEntity getHDFSPathEntity(Path path) { - String strPath = path.toString().toLowerCase(); + String strPath = path.toString(); + String name = Path.getPathWithoutSchemeAndAuthority(path).toString(); + + if (strPath.startsWith(HDFS_PATH_PREFIX) && context.isConvertHdfsPathToLowerCase()) { + strPath = strPath.toLowerCase(); + name = name.toLowerCase(); + } + String pathQualifiedName = getQualifiedName(strPath); AtlasEntity ret = context.getEntity(pathQualifiedName); @@ -490,7 +498,7 @@ public abstract class BaseHiveEvent { ret.setAttribute(ATTRIBUTE_PATH, strPath); ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, pathQualifiedName); - ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase()); + ret.setAttribute(ATTRIBUTE_NAME, name); ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, getClusterName()); context.putEntity(pathQualifiedName, ret); @@ -625,14 +633,18 @@ public abstract class BaseHiveEvent { } protected String getQualifiedName(URI location) { - String strPath = new Path(location).toString().toLowerCase(); + String strPath = new Path(location).toString(); + + if (strPath.startsWith(HDFS_PATH_PREFIX) && context.isConvertHdfsPathToLowerCase()) { + strPath = strPath.toLowerCase(); + } return getQualifiedName(strPath); } protected String getQualifiedName(String path) { - if (path.startsWith("hdfs://")) { - return (path + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); + if (path.startsWith(HDFS_PATH_PREFIX)) { + return path + QNAME_SEP_CLUSTER_NAME + getClusterName(); } return path.toLowerCase();