atlas git commit: ATLAS-2760: Hive hook updates to handle references to s3 paths

2018-06-20 Thread madhan 
Repository: atlas
Updated Branches:
  refs/heads/branch-1.0 bfd88c3c3 -> 674a46d0e


ATLAS-2760: Hive hook updates to handle references to s3 paths

(cherry picked from commit 88ac0fa62186a59ed08a950385c5a1e3c84e517e)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/674a46d0
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/674a46d0
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/674a46d0

Branch: refs/heads/branch-1.0
Commit: 674a46d0e16b6de052544e48d9b84a3ed811e8cb
Parents: bfd88c3
Author: Madhan Neethiraj 
Authored: Sun Jun 17 13:20:37 2018 -0700
Committer: Madhan Neethiraj 
Committed: Wed Jun 20 08:38:45 2018 -0700

--
 .../atlas/hive/hook/events/BaseHiveEvent.java   | 79 
 .../atlas/hive/hook/events/CreateTable.java |  2 +-
 2 files changed, 64 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/674a46d0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
--
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index ce03287..09f011c 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -78,6 +78,13 @@ public abstract class BaseHiveEvent {
 public static final String HDFS_TYPE_PATH   = "hdfs_path";
 public static final String HBASE_TYPE_TABLE = "hbase_table";
 public static final String HBASE_TYPE_NAMESPACE = "hbase_namespace";
+public static final String AWS_S3_BUCKET= "aws_s3_bucket";
+public static final String AWS_S3_PSEUDO_DIR= "aws_s3_pseudo_dir";
+public static final String AWS_S3_OBJECT= "aws_s3_object";
+
+public static final String SCHEME_SEPARATOR = "://";
+public static final String S3_SCHEME= "s3" + 
SCHEME_SEPARATOR;
+public static final String S3A_SCHEME   = "s3a" + 
SCHEME_SEPARATOR;
 
 public static final String ATTRIBUTE_QUALIFIED_NAME= 
"qualifiedName";
 public static final String ATTRIBUTE_NAME  = "name";
@@ -130,6 +137,8 @@ public abstract class BaseHiveEvent {
 public static final String ATTRIBUTE_URI   = "uri";
 public static final String ATTRIBUTE_STORAGE_HANDLER   = 
"storage_handler";
 public static final String ATTRIBUTE_NAMESPACE = 
"namespace";
+public static final String ATTRIBUTE_OBJECT_PREFIX = 
"objectPrefix";
+public static final String ATTRIBUTE_BUCKET= "bucket";
 
 public static final String HBASE_STORAGE_HANDLER_CLASS = 
"org.apache.hadoop.hive.hbase.HBaseStorageHandler";
 public static final String HBASE_DEFAULT_NAMESPACE = "default";
@@ -245,7 +254,7 @@ public abstract class BaseHiveEvent {
 URI location = entity.getLocation();
 
 if (location != null) {
-ret = getHDFSPathEntity(new Path(entity.getLocation()));
+ret = getPathEntity(new Path(entity.getLocation()), 
entityExtInfo);
 }
 }
 break;
@@ -494,26 +503,60 @@ public abstract class BaseHiveEvent {
 return ret;
 }
 
-protected AtlasEntity getHDFSPathEntity(Path path) {
-String  strPath   = path.toString().toLowerCase();
-String  nameServiceID = 
HdfsNameServiceResolver.getNameServiceIDForPath(strPath);
-String  attrPath  = StringUtils.isEmpty(nameServiceID) ? 
strPath : HdfsNameServiceResolver.getPathWithNameServiceID(strPath);
-String  pathQualifiedName = getQualifiedName(attrPath);
-AtlasEntity ret   = context.getEntity(pathQualifiedName);
+protected AtlasEntity getPathEntity(Path path, AtlasEntityExtInfo extInfo) 
{
+AtlasEntity ret;
+String  strPath = path.toString().toLowerCase();
 
-if (ret == null) {
-ret = new AtlasEntity(HDFS_TYPE_PATH);
+if (isS3Path(strPath)) {
+String  bucketName  = path.toUri().getAuthority();
+String  bucketQualifiedName = (path.toUri().getScheme() + 
SCHEME_SEPARATOR + path.toUri().getAuthority() + 
QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName();
+String  pathQualifiedName   = (strPath + 
QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName();
+AtlasEntity bucketEntity=

atlas git commit: ATLAS-2760: Hive hook updates to handle references to s3 paths

2018-06-20 Thread madhan 
Repository: atlas
Updated Branches:
  refs/heads/master f787bcc2b -> 88ac0fa62


ATLAS-2760: Hive hook updates to handle references to s3 paths


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/88ac0fa6
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/88ac0fa6
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/88ac0fa6

Branch: refs/heads/master
Commit: 88ac0fa62186a59ed08a950385c5a1e3c84e517e
Parents: f787bcc
Author: Madhan Neethiraj 
Authored: Sun Jun 17 13:20:37 2018 -0700
Committer: Madhan Neethiraj 
Committed: Tue Jun 19 23:26:04 2018 -0700

--
 .../atlas/hive/hook/events/BaseHiveEvent.java   | 79 
 .../atlas/hive/hook/events/CreateTable.java |  2 +-
 2 files changed, 64 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/88ac0fa6/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
--
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index ce03287..09f011c 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -78,6 +78,13 @@ public abstract class BaseHiveEvent {
 public static final String HDFS_TYPE_PATH   = "hdfs_path";
 public static final String HBASE_TYPE_TABLE = "hbase_table";
 public static final String HBASE_TYPE_NAMESPACE = "hbase_namespace";
+public static final String AWS_S3_BUCKET= "aws_s3_bucket";
+public static final String AWS_S3_PSEUDO_DIR= "aws_s3_pseudo_dir";
+public static final String AWS_S3_OBJECT= "aws_s3_object";
+
+public static final String SCHEME_SEPARATOR = "://";
+public static final String S3_SCHEME= "s3" + 
SCHEME_SEPARATOR;
+public static final String S3A_SCHEME   = "s3a" + 
SCHEME_SEPARATOR;
 
 public static final String ATTRIBUTE_QUALIFIED_NAME= 
"qualifiedName";
 public static final String ATTRIBUTE_NAME  = "name";
@@ -130,6 +137,8 @@ public abstract class BaseHiveEvent {
 public static final String ATTRIBUTE_URI   = "uri";
 public static final String ATTRIBUTE_STORAGE_HANDLER   = 
"storage_handler";
 public static final String ATTRIBUTE_NAMESPACE = 
"namespace";
+public static final String ATTRIBUTE_OBJECT_PREFIX = 
"objectPrefix";
+public static final String ATTRIBUTE_BUCKET= "bucket";
 
 public static final String HBASE_STORAGE_HANDLER_CLASS = 
"org.apache.hadoop.hive.hbase.HBaseStorageHandler";
 public static final String HBASE_DEFAULT_NAMESPACE = "default";
@@ -245,7 +254,7 @@ public abstract class BaseHiveEvent {
 URI location = entity.getLocation();
 
 if (location != null) {
-ret = getHDFSPathEntity(new Path(entity.getLocation()));
+ret = getPathEntity(new Path(entity.getLocation()), 
entityExtInfo);
 }
 }
 break;
@@ -494,26 +503,60 @@ public abstract class BaseHiveEvent {
 return ret;
 }
 
-protected AtlasEntity getHDFSPathEntity(Path path) {
-String  strPath   = path.toString().toLowerCase();
-String  nameServiceID = 
HdfsNameServiceResolver.getNameServiceIDForPath(strPath);
-String  attrPath  = StringUtils.isEmpty(nameServiceID) ? 
strPath : HdfsNameServiceResolver.getPathWithNameServiceID(strPath);
-String  pathQualifiedName = getQualifiedName(attrPath);
-AtlasEntity ret   = context.getEntity(pathQualifiedName);
+protected AtlasEntity getPathEntity(Path path, AtlasEntityExtInfo extInfo) 
{
+AtlasEntity ret;
+String  strPath = path.toString().toLowerCase();
 
-if (ret == null) {
-ret = new AtlasEntity(HDFS_TYPE_PATH);
+if (isS3Path(strPath)) {
+String  bucketName  = path.toUri().getAuthority();
+String  bucketQualifiedName = (path.toUri().getScheme() + 
SCHEME_SEPARATOR + path.toUri().getAuthority() + 
QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName();
+String  pathQualifiedName   = (strPath + 
QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName();
+AtlasEntity bucketEntity= 
context.getEntity(bucketQualifiedName);
+
+ret = context.getEntity(pathQualifiedName);
+
+