Repository: incubator-atlas Updated Branches: refs/heads/master fdf97ae4d -> aa15cd0ae
ATLAS-1139 Parameter name of a HDFS DataSet entity should contain filesystem path (svimal2106 via sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/aa15cd0a Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/aa15cd0a Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/aa15cd0a Branch: refs/heads/master Commit: aa15cd0aec15000a225f2a32477c4158d224b24e Parents: fdf97ae Author: Suma Shivaprasad <[email protected]> Authored: Mon Oct 24 15:26:10 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Mon Oct 24 15:26:10 2016 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/atlas/falcon/bridge/FalconBridge.java | 4 +++- .../java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java | 2 +- .../src/test/java/org/apache/atlas/hive/HiveITBase.java | 5 +++-- .../src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java | 4 ++-- .../main/java/org/apache/atlas/storm/hook/StormAtlasHook.java | 2 +- release-log.txt | 1 + 6 files changed, 11 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java index 919bd39..05d072f 100644 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java @@ -46,6 +46,7 @@ import org.apache.falcon.entity.v0.process.Input; import org.apache.falcon.entity.v0.process.Output; import org.apache.falcon.entity.v0.process.Workflow; import org.apache.falcon.workflow.WorkflowExecutionArgs; +import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -323,7 +324,8 @@ public class FalconBridge { // Path path = new Path(pathUri); // ref.set("name", path.getName()); //TODO - Fix after ATLAS-542 to shorter Name - ref.set(AtlasClient.NAME, pathUri); + Path path = new Path(pathUri); + ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase()); ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri); ref.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName); entities.add(ref); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index 31ce140..2d2039b 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -563,7 +563,7 @@ public class HiveMetaStoreBridge { Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString()); ref.set("path", pathUri); Path path = new Path(pathUri); - ref.set(AtlasClient.NAME, path.getName()); + ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase()); ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri); return ref; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java index bc3312d..bdf0d2d 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java @@ -218,13 +218,14 @@ public class HiveITBase { List<Id> hdfsPathRefs = (List<Id>) processReference.get(attributeName); for (int i = 0; i < testPaths.length; i++) { - final String testPathNormed = lower(new Path(testPaths[i]).toString()); + final Path path = new Path(testPaths[i]); + final String testPathNormed = lower(path.toString()); String hdfsPathId = assertHDFSPathIsRegistered(testPathNormed); Assert.assertEquals(hdfsPathRefs.get(0)._getId(), hdfsPathId); Referenceable hdfsPathRef = atlasClient.getEntity(hdfsPathId); Assert.assertEquals(hdfsPathRef.get("path"), testPathNormed); - Assert.assertEquals(hdfsPathRef.get(NAME), new Path(testPathNormed).getName()); + Assert.assertEquals(hdfsPathRef.get(NAME), Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase()); Assert.assertEquals(hdfsPathRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), testPathNormed); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java index 772f1b2..d4fb3af 100644 --- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java +++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java @@ -119,8 +119,8 @@ public class SqoopHookIT { private String assertHiveTableIsRegistered(String dbName, String tableName) throws Exception { LOG.debug("Searching for table {}.{}", dbName, tableName); String query = String.format( - "%s as t where " + AtlasClient.NAME + " = '%s', db where " + AtlasClient.NAME + " = '%s' and clusterName = '%s'" + " select t", - HiveDataTypes.HIVE_TABLE.getName(), tableName.toLowerCase(), dbName.toLowerCase(), CLUSTER_NAME); + "%s as t where " + AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME + " = '%s', db where " + AtlasClient.NAME + " = '%s' and clusterName = '%s'" + " select t", + HiveDataTypes.HIVE_TABLE.getName(), HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, dbName, tableName), dbName.toLowerCase(), CLUSTER_NAME); return assertEntityIsRegistered(query); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java index 89e1097..bb6a476 100644 --- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java +++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java @@ -223,7 +223,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook { dataSetReferenceable.set("path", hdfsPathStr); dataSetReferenceable.set(AtlasClient.OWNER, stormConf.get("hdfs.kerberos.principal")); final Path hdfsPath = new Path(hdfsPathStr); - dataSetReferenceable.set(AtlasClient.NAME, hdfsPath.getName()); + dataSetReferenceable.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(hdfsPath).toString().toLowerCase()); break; case "HiveBolt": http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/aa15cd0a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 9d29bb9..76c6a77 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1139 Parameter name of a HDFS DataSet entity should contain filesystem path (svimal2106 via sumasai) ATLAS-1200 Error Catalog enhancement (apoorvnaik via sumasai) ATLAS-1207 Dataset exists query in lineage APIs takes longer (shwethags) ATLAS-1232 added preCreate(), preDelete() in typedef persistence, to enable edge creation for references in a later stage (mneethiraj)
