This is an automated email from the ASF dual-hosted git repository. amestry pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push: new 8b58a32 ATLAS-3182: Import transforms handling qualifiedName for hive_table entity. 8b58a32 is described below commit 8b58a326197e897a937d28f53076f385bb6710a7 Author: Ashutosh Mestry <ames...@hortonworks.com> AuthorDate: Thu May 2 21:13:39 2019 -0700 ATLAS-3182: Import transforms handling qualifiedName for hive_table entity. --- .../entitytransform/HiveTableEntityHandler.java | 19 +++- .../entitytransform/TransformationHandlerTest.java | 104 +++++++++++++++------ intg/src/test/resources/json/entity1.json | 48 ++++++++++ intg/src/test/resources/json/transform1.json | 1 + 4 files changed, 139 insertions(+), 33 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java index 9eb44d7..caf08dd 100644 --- a/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java +++ b/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java @@ -43,18 +43,21 @@ public class HiveTableEntityHandler extends BaseEntityHandler { } private static class HiveTableEntity extends AtlasTransformableEntity { - private String databaseName; - private String tableName; - private String clusterName; - private boolean isCustomAttributeUpdated = false; + private final String qualifiedName; + private String databaseName; + private String tableName; + private String clusterName; + private boolean isCustomAttributeUpdated = false; + private final boolean tableNameFromQualifiedNameDifferent; + private final String tableNameFromQualifiedName; public HiveTableEntity(AtlasEntity entity) { super(entity); this.tableName = (String) entity.getAttribute(NAME_ATTRIBUTE); - String qualifiedName = (String) entity.getAttribute(QUALIFIED_NAME_ATTRIBUTE); + this.qualifiedName = (String) entity.getAttribute(QUALIFIED_NAME_ATTRIBUTE); if (qualifiedName != null) { int databaseSeparatorIdx = qualifiedName.indexOf(DATABASE_DELIMITER); @@ -62,9 +65,13 @@ public class HiveTableEntityHandler extends BaseEntityHandler { this.databaseName = databaseSeparatorIdx != -1 ? qualifiedName.substring(0, databaseSeparatorIdx) : ""; this.clusterName = clusterSeparatorIdx != -1 ? qualifiedName.substring(clusterSeparatorIdx + 1) : ""; + this.tableNameFromQualifiedName = clusterSeparatorIdx != -1 ? this.qualifiedName.substring(databaseSeparatorIdx + 1, clusterSeparatorIdx) : ""; + this.tableNameFromQualifiedNameDifferent = !this.tableNameFromQualifiedName.equals(this.tableName); } else { this.databaseName = ""; this.clusterName = ""; + this.tableNameFromQualifiedName = ""; + this.tableNameFromQualifiedNameDifferent = false; } } @@ -121,6 +128,8 @@ public class HiveTableEntityHandler extends BaseEntityHandler { private String toQualifiedName() { + String tableName = tableNameFromQualifiedNameDifferent ? this.tableNameFromQualifiedName : this.tableName; + return String.format("%s.%s@%s", databaseName, tableName, clusterName); } } diff --git a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java index d6b0ede..d3f4b57 100644 --- a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java +++ b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java @@ -24,13 +24,16 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.Test; +import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -42,6 +45,7 @@ import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_COLUMN; import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DATABASE; import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_STORAGE_DESCRIPTOR; +import static org.apache.atlas.entitytransform.TransformationConstants.QUALIFIED_NAME_ATTRIBUTE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; @@ -59,6 +63,7 @@ public class TransformationHandlerTest { private static final String[] DATABASE_NAMES = new String[] { "hr", "sales", "engg" }; private static final String[] TABLE_NAMES = new String[] { "employees", "products", "invoice" }; private static final String[] COLUMN_NAMES = new String[] { "name", "age", "dob" }; + private final String ATTR_NAME_QUALIFIED_NAME = "qualifiedName"; @Test public void testHdfsClusterRenameHandler() { @@ -69,12 +74,12 @@ public class TransformationHandlerTest { List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p1)); for (AtlasEntity hdfsPath : getHdfsPathEntities()) { - String qualifiedName = (String) hdfsPath.getAttribute("qualifiedName"); + String qualifiedName = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME); boolean endsWithCl1 = qualifiedName.endsWith("@cl1"); applyTransforms(hdfsPath, handlers); - String transformedValue = (String) hdfsPath.getAttribute("qualifiedName"); + String transformedValue = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME); if (endsWithCl1) { assertTrue(transformedValue.endsWith("@cl2"), transformedValue + ": expected to end with @cl2"); @@ -95,12 +100,12 @@ public class TransformationHandlerTest { List<AtlasEntity> hdfsPaths = getHdfsPathEntities(); for (AtlasEntity hdfsPath : hdfsPaths) { - String qualifiedName = (String) hdfsPath.getAttribute("qualifiedName"); + String qualifiedName = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME); boolean endsWithCl1 = qualifiedName.endsWith("@cl1"); applyTransforms(hdfsPath, handlers); - String transformedValue = (String) hdfsPath.getAttribute("qualifiedName"); + String transformedValue = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME); if (endsWithCl1) { assertTrue(transformedValue.endsWith("@CL1"), transformedValue + ": expected to end with @CL1"); @@ -116,12 +121,12 @@ public class TransformationHandlerTest { handlers = initializeHandlers(Collections.singletonList(p2)); for (AtlasEntity hdfsPath : hdfsPaths) { - String qualifiedName = (String) hdfsPath.getAttribute("qualifiedName"); + String qualifiedName = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME); boolean endsWithCL1 = qualifiedName.endsWith("@CL1"); applyTransforms(hdfsPath, handlers); - String transformedValue = (String) hdfsPath.getAttribute("qualifiedName"); + String transformedValue = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME); if (endsWithCL1) { assertTrue(transformedValue.endsWith("@cl1"), transformedValue + ": expected to end with @cl1"); @@ -258,14 +263,14 @@ public class TransformationHandlerTest { List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p1)); for (AtlasEntity entity : getAllEntities()) { - String qualifiedName = (String) entity.getAttribute("qualifiedName"); + String qualifiedName = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); boolean isHdfsPath = StringUtils.equals(entity.getTypeName(), HDFS_PATH); boolean endsWithCl1 = qualifiedName.endsWith("@cl1"); boolean containsCl1 = qualifiedName.contains("@cl1"); // for stroage_desc applyTransforms(entity, handlers); - String transformedValue = (String) entity.getAttribute("qualifiedName"); + String transformedValue = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); if (!isHdfsPath && endsWithCl1) { assertTrue(transformedValue.endsWith("@cl1_backup"), transformedValue + ": expected to end with @cl1_backup"); @@ -286,18 +291,18 @@ public class TransformationHandlerTest { List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p)); for (AtlasEntity entity : getAllEntities()) { - String qualifiedName = (String) entity.getAttribute("qualifiedName"); + String qualifiedName = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); boolean startsWithHrDot = qualifiedName.startsWith("hr."); // for tables, columns boolean startsWithHrAt = qualifiedName.startsWith("hr@"); // for databases applyTransforms(entity, handlers); if (startsWithHrDot) { - assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr_backup.")); + assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr_backup.")); } else if (startsWithHrAt) { - assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr_backup@")); + assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr_backup@")); } else { - assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change"); + assertEquals(qualifiedName, (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), "not expected to change"); } } } @@ -313,18 +318,18 @@ public class TransformationHandlerTest { List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p)); for (AtlasEntity entity : getAllEntities()) { - String qualifiedName = (String) entity.getAttribute("qualifiedName"); + String qualifiedName = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); boolean startsWithHrEmployeesDot = qualifiedName.startsWith("hr.employees."); // for columns boolean startsWithHrEmployeesAt = qualifiedName.startsWith("hr.employees@"); // for tables applyTransforms(entity, handlers); if (startsWithHrEmployeesDot) { - assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees_backup.")); + assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr.employees_backup.")); } else if (startsWithHrEmployeesAt) { - assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees_backup@")); + assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr.employees_backup@")); } else { - assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change"); + assertEquals(qualifiedName, (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), "not expected to change"); } } } @@ -341,15 +346,15 @@ public class TransformationHandlerTest { List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p)); for (AtlasEntity entity : getAllEntities()) { - String qualifiedName = (String) entity.getAttribute("qualifiedName"); + String qualifiedName = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); boolean startsWithHrEmployeesAgeAt = qualifiedName.startsWith("hr.employees.age@"); applyTransforms(entity, handlers); if (startsWithHrEmployeesAgeAt) { - assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees.age_backup@")); + assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr.employees.age_backup@")); } else { - assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change"); + assertEquals(qualifiedName, (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), "not expected to change"); } } } @@ -369,7 +374,7 @@ public class TransformationHandlerTest { public void verifyAddClassificationUsingScope() { AtlasExportRequest exportRequest = new AtlasExportRequest(); - exportRequest.setItemsToExport(Collections.singletonList(new AtlasObjectId("hive_db", Collections.singletonMap("qualifiedName", "hr@cl1")))); + exportRequest.setItemsToExport(Collections.singletonList(new AtlasObjectId("hive_db", Collections.singletonMap(ATTR_NAME_QUALIFIED_NAME, "hr@cl1")))); AtlasEntityTransformer transformer = new AtlasEntityTransformer(Collections.singletonMap("Referenceable.", "topLevel: "), Collections.singletonMap("Referenceable", "ADD_CLASSIFICATION: replicated"), @@ -420,11 +425,40 @@ public class TransformationHandlerTest { assertEquals((String) nonAssetEntity.getAttribute("name"), "renamed", "name expected to be not updated for non-Asset type entity"); } + @Test + public void qualifiedNameDifferentFromName() throws IOException { + final String expectedQualifiedName = "test_partition_bootstrap_target.temptable_temp-5e93084a-f2d1-46f7-8f8f-30ffb859d2be@mycluster1"; + + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = readObjectFromJson("entity1", AtlasEntity.AtlasEntityWithExtInfo.class); + List<BaseEntityHandler> handlers = BaseEntityHandler.fromJson(getStringFromFile("transform1"), null); + + assertNotNull(handlers); + assertEquals(handlers.size(), 4); + assertNotNull(entityWithExtInfo); + + applyTransforms(entityWithExtInfo.getEntity(), handlers); + assertEquals(entityWithExtInfo.getEntity().getAttribute(ATTR_NAME_QUALIFIED_NAME), expectedQualifiedName); + } + + @Test + public void emptyQualifiedNameHandling() throws IOException { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = readObjectFromJson("entity1", AtlasEntity.AtlasEntityWithExtInfo.class); + entityWithExtInfo.getEntity().setAttribute(QUALIFIED_NAME_ATTRIBUTE, ""); + List<BaseEntityHandler> handlers = BaseEntityHandler.fromJson("[{\"conditions\":{\"hive_db.clusterName\":\"EQUALS: mycluster0\"},\"action\":{\"hive_db.clusterName\":\"SET: mycluster1\"}}]", null); + + assertNotNull(handlers); + assertEquals(handlers.size(), 4); + assertNotNull(entityWithExtInfo); + + applyTransforms(entityWithExtInfo.getEntity(), handlers); + assertEquals(entityWithExtInfo.getEntity().getAttribute(ATTR_NAME_QUALIFIED_NAME), ""); + } + private void assertApplyTransform(List<BaseEntityHandler> handlers) { for (AtlasEntity entity : getAllEntities()) { applyTransforms(entity, handlers); - if(entity.getTypeName().equals("hive_db") && entity.getAttribute("qualifiedName").equals("hr@cl1")) { + if(entity.getTypeName().equals("hive_db") && entity.getAttribute(ATTR_NAME_QUALIFIED_NAME).equals("hr@cl1")) { assertNotNull(entity.getClassifications()); } else{ assertNull(entity.getClassifications()); @@ -522,7 +556,7 @@ public class TransformationHandlerTest { entity.setAttribute("name", "/aa/bb/employee"); entity.setAttribute("path", "hdfs://localhost.localdomain:8020/aa/bb/employee"); - entity.setAttribute("qualifiedName", "hdfs://localhost.localdomain:8020/aa/bb/employee@" + clusterName); + entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, "hdfs://localhost.localdomain:8020/aa/bb/employee@" + clusterName); entity.setAttribute("clusterName", clusterName); entity.setAttribute("isSymlink", false); entity.setAttribute("modifiedTime", 0); @@ -539,7 +573,7 @@ public class TransformationHandlerTest { entity.setAttribute("name", "/cc/dd/employee"); entity.setAttribute("path", "hdfs://localhost.localdomain:8020/cc/dd/employee"); - entity.setAttribute("qualifiedName", "hdfs://localhost.localdomain:8020/cc/dd/employee@" + clusterName); + entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, "hdfs://localhost.localdomain:8020/cc/dd/employee@" + clusterName); entity.setAttribute("clusterName", clusterName); entity.setAttribute("isSymlink", false); entity.setAttribute("modifiedTime", 0); @@ -555,7 +589,7 @@ public class TransformationHandlerTest { AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_DATABASE); entity.setAttribute("name", dbName); - entity.setAttribute("qualifiedName", dbName + "@" + clusterName); + entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, dbName + "@" + clusterName); entity.setAttribute("location", "hdfs://localhost.localdomain:8020/warehouse/tablespace/managed/hive/" + dbName + ".db"); entity.setAttribute("clusterName", clusterName); entity.setAttribute("owner", "hive"); @@ -570,7 +604,7 @@ public class TransformationHandlerTest { AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_TABLE); entity.setAttribute("name", tableName); - entity.setAttribute("qualifiedName", qualifiedName); + entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, qualifiedName); entity.setAttribute("owner", "hive"); entity.setAttribute("temporary", false); entity.setAttribute("lastAccessTime", "1535656355000"); @@ -588,7 +622,7 @@ public class TransformationHandlerTest { AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_STORAGE_DESCRIPTOR); - entity.setAttribute("qualifiedName", dbName + "." + tableName + "@" + clusterName + "_storage"); + entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, dbName + "." + tableName + "@" + clusterName + "_storage"); entity.setAttribute("storedAsSubDirectories", false); entity.setAttribute("location", qualifiedName); entity.setAttribute("compressed", false); @@ -605,7 +639,7 @@ public class TransformationHandlerTest { AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_COLUMN); entity.setAttribute("owner", "hive"); - entity.setAttribute("qualifiedName", qualifiedName); + entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, qualifiedName); entity.setAttribute("name", columnName); entity.setAttribute("position", 1); entity.setAttribute("type", "string"); @@ -616,4 +650,18 @@ public class TransformationHandlerTest { private AtlasEntity getNonAssetEntity() { return new AtlasEntity(TYPENAME_NON_ASSET); } + + public static <T> T readObjectFromJson(String filename, Class<T> objectClass) throws IOException { + String json = getStringFromFile(filename); + return AtlasType.fromJson(json, objectClass); + } + + private static String getStringFromFile(String filename) throws IOException { + final String userDir = System.getProperty("user.dir"); + return FileUtils.readFileToString(new File(getTestJsonPath(userDir, filename))); + } + + private static String getTestJsonPath(String startPath, String fileName) { + return startPath + "/src/test/resources/json/" + fileName + ".json"; + } } \ No newline at end of file diff --git a/intg/src/test/resources/json/entity1.json b/intg/src/test/resources/json/entity1.json new file mode 100644 index 0000000..85aa62b --- /dev/null +++ b/intg/src/test/resources/json/entity1.json @@ -0,0 +1,48 @@ +{ + "entity": { + "attributes": { + "aliases": null, + "columns": [], + "comment": null, + "createTime": 1556559566000, + "db": { + "guid": "9ef83dcb-6cca-4e10-a19c-752d64227a94", + "typeName": "hive_db" + }, + "description": null, + "lastAccessTime": 1556559566000, + "name": "temptable", + "owner": "hrt_qa", + "parameters": { + "bucketing_version": "2", + "numFiles": "1", + "numRows": "0", + "rawDataSize": "0", + "totalSize": "378890" + }, + "partitionKeys": [], + "qualifiedName": "test_partition_bootstrap.temptable_temp-5e93084a-f2d1-46f7-8f8f-30ffb859d2be@mycluster0", + "replicatedFrom": null, + "replicatedTo": [ + { + "guid": "2306c3e3-8b92-418e-b788-ade3d82d68db", + "typeName": "AtlasServer" + } + ], + "retention": 0, + "tableType": "MANAGED_TABLE", + "temporary": true, + "viewExpandedText": null, + "viewOriginalText": null + }, + "createTime": 1556559566442, + "createdBy": "hrt_qa", + "guid": "e2d30fc3-026e-4597-b6a4-c706066bd480", + "status": "ACTIVE", + "typeName": "hive_table", + "updateTime": 1556559585192, + "updatedBy": "hrt_qa", + "version": 0 + }, + "referredEntities": {} +} diff --git a/intg/src/test/resources/json/transform1.json b/intg/src/test/resources/json/transform1.json new file mode 100644 index 0000000..1093003 --- /dev/null +++ b/intg/src/test/resources/json/transform1.json @@ -0,0 +1 @@ +[{"conditions":{"hive_db.clusterName":"EQUALS: mycluster0"},"action":{"hive_db.clusterName":"SET: mycluster1"}},{"conditions":{"hive_db.name":"EQUALS: test_partition_bootstrap"},"action":{"hive_db.name":"SET: test_partition_bootstrap_target"}},{"conditions":{"hive_db.location":"STARTS_WITH_IGNORE_CASE: hdfs://ctr-e139-1542663976389-101273-01-000011.hwx.site:8020"},"action":{"hive_db.location":"REPLACE_PREFIX: = :hdfs://ctr-e139-1542663976389-101273-01-000011.hwx.site:8020=hdfs://ctr-e139 [...] \ No newline at end of file