atlas git commit: ATLAS-2760: Hive hook updates to handle references to s3 paths
Repository: atlas Updated Branches: refs/heads/branch-1.0 bfd88c3c3 -> 674a46d0e ATLAS-2760: Hive hook updates to handle references to s3 paths (cherry picked from commit 88ac0fa62186a59ed08a950385c5a1e3c84e517e) Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/674a46d0 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/674a46d0 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/674a46d0 Branch: refs/heads/branch-1.0 Commit: 674a46d0e16b6de052544e48d9b84a3ed811e8cb Parents: bfd88c3 Author: Madhan Neethiraj Authored: Sun Jun 17 13:20:37 2018 -0700 Committer: Madhan Neethiraj Committed: Wed Jun 20 08:38:45 2018 -0700 -- .../atlas/hive/hook/events/BaseHiveEvent.java | 79 .../atlas/hive/hook/events/CreateTable.java | 2 +- 2 files changed, 64 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/atlas/blob/674a46d0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java -- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java index ce03287..09f011c 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java @@ -78,6 +78,13 @@ public abstract class BaseHiveEvent { public static final String HDFS_TYPE_PATH = "hdfs_path"; public static final String HBASE_TYPE_TABLE = "hbase_table"; public static final String HBASE_TYPE_NAMESPACE = "hbase_namespace"; +public static final String AWS_S3_BUCKET= "aws_s3_bucket"; +public static final String AWS_S3_PSEUDO_DIR= "aws_s3_pseudo_dir"; +public static final String AWS_S3_OBJECT= "aws_s3_object"; + +public static final String SCHEME_SEPARATOR = "://"; +public static final String S3_SCHEME= "s3" + SCHEME_SEPARATOR; +public static final String S3A_SCHEME = "s3a" + SCHEME_SEPARATOR; public static final String ATTRIBUTE_QUALIFIED_NAME= "qualifiedName"; public static final String ATTRIBUTE_NAME = "name"; @@ -130,6 +137,8 @@ public abstract class BaseHiveEvent { public static final String ATTRIBUTE_URI = "uri"; public static final String ATTRIBUTE_STORAGE_HANDLER = "storage_handler"; public static final String ATTRIBUTE_NAMESPACE = "namespace"; +public static final String ATTRIBUTE_OBJECT_PREFIX = "objectPrefix"; +public static final String ATTRIBUTE_BUCKET= "bucket"; public static final String HBASE_STORAGE_HANDLER_CLASS = "org.apache.hadoop.hive.hbase.HBaseStorageHandler"; public static final String HBASE_DEFAULT_NAMESPACE = "default"; @@ -245,7 +254,7 @@ public abstract class BaseHiveEvent { URI location = entity.getLocation(); if (location != null) { -ret = getHDFSPathEntity(new Path(entity.getLocation())); +ret = getPathEntity(new Path(entity.getLocation()), entityExtInfo); } } break; @@ -494,26 +503,60 @@ public abstract class BaseHiveEvent { return ret; } -protected AtlasEntity getHDFSPathEntity(Path path) { -String strPath = path.toString().toLowerCase(); -String nameServiceID = HdfsNameServiceResolver.getNameServiceIDForPath(strPath); -String attrPath = StringUtils.isEmpty(nameServiceID) ? strPath : HdfsNameServiceResolver.getPathWithNameServiceID(strPath); -String pathQualifiedName = getQualifiedName(attrPath); -AtlasEntity ret = context.getEntity(pathQualifiedName); +protected AtlasEntity getPathEntity(Path path, AtlasEntityExtInfo extInfo) { +AtlasEntity ret; +String strPath = path.toString().toLowerCase(); -if (ret == null) { -ret = new AtlasEntity(HDFS_TYPE_PATH); +if (isS3Path(strPath)) { +String bucketName = path.toUri().getAuthority(); +String bucketQualifiedName = (path.toUri().getScheme() + SCHEME_SEPARATOR + path.toUri().getAuthority() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); +String pathQualifiedName = (strPath + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); +AtlasEntity bucketEntity= context.getEntity(bucketQual
atlas git commit: ATLAS-2760: Hive hook updates to handle references to s3 paths
Repository: atlas Updated Branches: refs/heads/master f787bcc2b -> 88ac0fa62 ATLAS-2760: Hive hook updates to handle references to s3 paths Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/88ac0fa6 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/88ac0fa6 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/88ac0fa6 Branch: refs/heads/master Commit: 88ac0fa62186a59ed08a950385c5a1e3c84e517e Parents: f787bcc Author: Madhan Neethiraj Authored: Sun Jun 17 13:20:37 2018 -0700 Committer: Madhan Neethiraj Committed: Tue Jun 19 23:26:04 2018 -0700 -- .../atlas/hive/hook/events/BaseHiveEvent.java | 79 .../atlas/hive/hook/events/CreateTable.java | 2 +- 2 files changed, 64 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/atlas/blob/88ac0fa6/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java -- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java index ce03287..09f011c 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java @@ -78,6 +78,13 @@ public abstract class BaseHiveEvent { public static final String HDFS_TYPE_PATH = "hdfs_path"; public static final String HBASE_TYPE_TABLE = "hbase_table"; public static final String HBASE_TYPE_NAMESPACE = "hbase_namespace"; +public static final String AWS_S3_BUCKET= "aws_s3_bucket"; +public static final String AWS_S3_PSEUDO_DIR= "aws_s3_pseudo_dir"; +public static final String AWS_S3_OBJECT= "aws_s3_object"; + +public static final String SCHEME_SEPARATOR = "://"; +public static final String S3_SCHEME= "s3" + SCHEME_SEPARATOR; +public static final String S3A_SCHEME = "s3a" + SCHEME_SEPARATOR; public static final String ATTRIBUTE_QUALIFIED_NAME= "qualifiedName"; public static final String ATTRIBUTE_NAME = "name"; @@ -130,6 +137,8 @@ public abstract class BaseHiveEvent { public static final String ATTRIBUTE_URI = "uri"; public static final String ATTRIBUTE_STORAGE_HANDLER = "storage_handler"; public static final String ATTRIBUTE_NAMESPACE = "namespace"; +public static final String ATTRIBUTE_OBJECT_PREFIX = "objectPrefix"; +public static final String ATTRIBUTE_BUCKET= "bucket"; public static final String HBASE_STORAGE_HANDLER_CLASS = "org.apache.hadoop.hive.hbase.HBaseStorageHandler"; public static final String HBASE_DEFAULT_NAMESPACE = "default"; @@ -245,7 +254,7 @@ public abstract class BaseHiveEvent { URI location = entity.getLocation(); if (location != null) { -ret = getHDFSPathEntity(new Path(entity.getLocation())); +ret = getPathEntity(new Path(entity.getLocation()), entityExtInfo); } } break; @@ -494,26 +503,60 @@ public abstract class BaseHiveEvent { return ret; } -protected AtlasEntity getHDFSPathEntity(Path path) { -String strPath = path.toString().toLowerCase(); -String nameServiceID = HdfsNameServiceResolver.getNameServiceIDForPath(strPath); -String attrPath = StringUtils.isEmpty(nameServiceID) ? strPath : HdfsNameServiceResolver.getPathWithNameServiceID(strPath); -String pathQualifiedName = getQualifiedName(attrPath); -AtlasEntity ret = context.getEntity(pathQualifiedName); +protected AtlasEntity getPathEntity(Path path, AtlasEntityExtInfo extInfo) { +AtlasEntity ret; +String strPath = path.toString().toLowerCase(); -if (ret == null) { -ret = new AtlasEntity(HDFS_TYPE_PATH); +if (isS3Path(strPath)) { +String bucketName = path.toUri().getAuthority(); +String bucketQualifiedName = (path.toUri().getScheme() + SCHEME_SEPARATOR + path.toUri().getAuthority() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); +String pathQualifiedName = (strPath + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); +AtlasEntity bucketEntity= context.getEntity(bucketQualifiedName); + +ret = context.getEntity(pathQualifiedName); + +