This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 4e6339d ATLAS-3197: Disable creation of hive process, hive process
execution and ddl query entities for HMS hook
4e6339d is described below
commit 4e6339d8721e23bec7aabaafd28412497df2bb17
Author: Le Ma <[email protected]>
AuthorDate: Thu May 9 16:40:40 2019 -0700
ATLAS-3197: Disable creation of hive process, hive process execution and
ddl query entities for HMS hook
---
.../atlas/hive/hook/events/AlterTableRename.java | 8 ++-
.../atlas/hive/hook/events/CreateHiveProcess.java | 10 +++-
.../apache/atlas/hive/hook/events/CreateTable.java | 11 ++--
.../org/apache/atlas/hive/hook/HiveHookIT.java | 68 +++++++++++++++++-----
.../atlas/hive/hook/HiveMetastoreHookIT.java | 34 ++++++++---
5 files changed, 98 insertions(+), 33 deletions(-)
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
index 67ea527..927ddc0 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
@@ -148,10 +148,12 @@ public class AlterTableRename extends BaseHiveEvent {
ret.add(new EntityPartialUpdateRequestV2(getUserName(), oldTableId,
renamedTableEntity));
// partial update relationship attribute ddl
- AtlasEntity ddlEntity =
createHiveDDLEntity(renamedTableEntity.getEntity(), true);
+ if (!context.isMetastoreHook()) {
+ AtlasEntity ddlEntity =
createHiveDDLEntity(renamedTableEntity.getEntity(), true);
- if (ddlEntity != null) {
- ret.add(new HookNotification.EntityCreateRequestV2(getUserName(),
new AtlasEntitiesWithExtInfo(ddlEntity)));
+ if (ddlEntity != null) {
+ ret.add(new
HookNotification.EntityCreateRequestV2(getUserName(), new
AtlasEntitiesWithExtInfo(ddlEntity)));
+ }
}
context.removeFromKnownTable(oldTableQualifiedName);
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
index d1511c0..38b53b6 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
@@ -108,6 +108,14 @@ public class CreateHiveProcess extends BaseHiveEvent {
if (entity != null) {
outputs.add(entity);
}
+
+ if (entity != null && !context.isMetastoreHook()) {
+ AtlasEntity ddlEntity = createHiveDDLEntity(entity);
+
+ if (ddlEntity != null) {
+ ret.addEntity(ddlEntity);
+ }
+ }
}
}
@@ -121,7 +129,7 @@ public class CreateHiveProcess extends BaseHiveEvent {
}
}
- if (!skipProcess) {
+ if (!skipProcess && !context.isMetastoreHook()) {
AtlasEntity process = getHiveProcessEntity(inputs, outputs);
ret.addEntity(process);
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
index ef4e687..aedb155 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
@@ -117,7 +117,7 @@ public class CreateTable extends BaseHiveEvent {
if (table != null) {
AtlasEntity tblEntity = toTableEntity(table, ret);
- if (tblEntity != null) {
+ if (tblEntity != null && !context.isMetastoreHook()) {
if (isHBaseStore(table)) {
// This create lineage to HBase table in case of Hive on
HBase
AtlasEntity hbaseTableEntity =
toReferencedHBaseTable(table, ret);
@@ -134,7 +134,6 @@ public class CreateTable extends BaseHiveEvent {
AtlasEntity processExecution =
getHiveProcessExecutionEntity(processEntity);
ret.addEntity(processExecution);
-
}
} else {
if (EXTERNAL_TABLE.equals(table.getTableType())) {
@@ -148,12 +147,12 @@ public class CreateTable extends BaseHiveEvent {
ret.addEntity(processExecution);
}
}
- }
- AtlasEntity tableDDLEntity = createHiveDDLEntity(tblEntity);
+ AtlasEntity tableDDLEntity = createHiveDDLEntity(tblEntity);
- if (tableDDLEntity != null) {
- ret.addEntity(tableDDLEntity);
+ if (tableDDLEntity != null) {
+ ret.addEntity(tableDDLEntity);
+ }
}
}
}
diff --git
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 28c600e..d19470f 100755
---
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -128,7 +128,7 @@ public class HiveHookIT extends HiveITBase {
String tableId = assertTableIsRegistered(dbName, tableName);
String colId =
assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
dbName, tableName), colName)); //there is only one instance of column
registered
AtlasEntity colEntity =
atlasClientV2.getEntityByGuid(colId).getEntity();
- AtlasEntity tblEntity =
atlasClientV2.getEntityByGuid(colId).getEntity();
+ AtlasEntity tblEntity =
atlasClientV2.getEntityByGuid(tableId).getEntity();
Assert.assertEquals(colEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME),
String.format("%s.%s.%s@%s", dbName.toLowerCase(), tableName.toLowerCase(),
colName.toLowerCase(), CLUSTER_NAME));
Assert.assertNotNull(colEntity.getAttribute(ATTRIBUTE_TABLE));
@@ -195,14 +195,19 @@ public class HiveHookIT extends HiveITBase {
//ATLAS-1321: Disable problematic tests. Need to revisit and fix them later
@Test(enabled = false)
public void testCreateExternalTable() throws Exception {
- String tableName = tableName();
- String colName = columnName();
- String pFile = createTestDFSPath("parentPath");
- String query = String.format("create EXTERNAL table %s.%s(%s, %s)
location '%s'", DEFAULT_DB , tableName , colName + " int", "name string",
pFile);
+ String tableName = tableName();
+ String colName = columnName();
+ String pFile = createTestDFSPath("parentPath");
+ String query = String.format("create EXTERNAL table %s.%s(%s,
%s) location '%s'", DEFAULT_DB , tableName , colName + " int", "name string",
pFile);
runCommand(query);
- assertTableIsRegistered(DEFAULT_DB, tableName, null, true);
+ String tblId = assertTableIsRegistered(DEFAULT_DB, tableName,
null, true);
+ AtlasEntity tblEnity =
atlasClientV2.getEntityByGuid(tblId).getEntity();
+ List ddlList = (List)
tblEnity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+ assertNotNull(ddlList);
+ assertEquals(ddlList.size(), 1);
String processId =
assertEntityIsRegistered(HiveDataTypes.HIVE_PROCESS.getName(),
ATTRIBUTE_QUALIFIED_NAME, getTableProcessQualifiedName(DEFAULT_DB, tableName),
null);
@@ -381,15 +386,20 @@ public class HiveHookIT extends HiveITBase {
BaseHiveEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process.getGuid(), processEntity1.getGuid());
- String drpquery = String.format("drop table %s ",
ctasTableName);
+ String dropQuery = String.format("drop table %s ",
ctasTableName);
- runCommandWithDelay(drpquery, 100);
+ runCommandWithDelay(dropQuery, 5000);
assertTableIsNotRegistered(DEFAULT_DB, ctasTableName);
runCommand(query);
- assertTableIsRegistered(DEFAULT_DB, ctasTableName);
+ String tblId = assertTableIsRegistered(DEFAULT_DB,
ctasTableName);
+ AtlasEntity tblEntity =
atlasClientV2.getEntityByGuid(tblId).getEntity();
+ List ddlList = (List)
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+ assertNotNull(ddlList);
+ assertEquals(ddlList.size(), 1);
outputs = getOutputs(ctasTableName, Entity.Type.TABLE);
@@ -423,6 +433,13 @@ public class HiveHookIT extends HiveITBase {
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
assertTableIsRegistered(DEFAULT_DB, viewName);
+
+ String viewId = assertTableIsRegistered(DEFAULT_DB, viewName);
+ AtlasEntity viewEntity =
atlasClientV2.getEntityByGuid(viewId).getEntity();
+ List ddlQueries = (List)
viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+ Assert.assertNotNull(ddlQueries);
+ Assert.assertEquals(ddlQueries.size(), 1);
}
@Test
@@ -476,8 +493,15 @@ public class HiveHookIT extends HiveITBase {
Assert.assertEquals(processEntity1.getGuid(),
processEntity2.getGuid());
String table2Id = assertTableIsRegistered(DEFAULT_DB, table2Name);
+ String viewId2 = assertTableIsRegistered(DEFAULT_DB, viewName);
- Assert.assertEquals(assertTableIsRegistered(DEFAULT_DB, viewName),
viewId);
+ Assert.assertEquals(viewId2, viewId);
+
+ AtlasEntity viewEntity =
atlasClientV2.getEntityByGuid(viewId2).getEntity();
+ List ddlQueries = (List)
viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+ Assert.assertNotNull(ddlQueries);
+ Assert.assertEquals(ddlQueries.size(), 2);
datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
DEFAULT_DB, viewName);
@@ -745,7 +769,13 @@ public class HiveHookIT extends HiveITBase {
Assert.assertEquals(process.getGuid(), hiveProcess.getGuid());
Assert.assertEquals(numberOfProcessExecutions(hiveProcess), 1);
- assertTableIsRegistered(DEFAULT_DB, tableName);
+ String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);
+
+ AtlasEntity tblEntity =
atlasClientV2.getEntityByGuid(tblId).getEntity();
+ List ddlQueries = (List)
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+ Assert.assertNotNull(ddlQueries);
+ Assert.assertEquals(ddlQueries.size(), 1);
}
@Test
@@ -797,7 +827,13 @@ public class HiveHookIT extends HiveITBase {
runCommandWithDelay(query, 1000);
- assertTableIsRegistered(DEFAULT_DB, tableName);
+ String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);
+
+ AtlasEntity tblEntity =
atlasClientV2.getEntityByGuid(tblId).getEntity();
+ List ddlQueries = (List)
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+ Assert.assertNotNull(ddlQueries);
+ Assert.assertEquals(ddlQueries.size(), 1);
Set<WriteEntity> p3Outputs = new LinkedHashSet<WriteEntity>() {{
addAll(getOutputs(pFile2, Entity.Type.DFS_DIR));
@@ -1308,9 +1344,9 @@ public class HiveHookIT extends HiveITBase {
assertEquals(columns.size(), 1);
assertEquals(columns.get(0).getAttribute(NAME), "name");
- String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);
+ String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);
AtlasEntity tblEntity =
atlasClientV2.getEntityByGuid(tblId).getEntity();
- List ddlQueries = (List)
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+ List ddlQueries = (List)
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertNotNull(ddlQueries);
Assert.assertEquals(ddlQueries.size(), 2);
@@ -1430,7 +1466,7 @@ public class HiveHookIT extends HiveITBase {
List ddlQueries3 = (List)
tblEntity3.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertNotNull(ddlQueries3);
- Assert.assertEquals(ddlQueries3.size(), 4);
+ Assert.assertEquals(ddlQueries3.size(), 5);
//Change col position again
oldColName = "name4";
@@ -1466,7 +1502,7 @@ public class HiveHookIT extends HiveITBase {
List ddlQueries4 = (List)
tblEntity4.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertNotNull(ddlQueries4);
- Assert.assertEquals(ddlQueries4.size(), 5);
+ Assert.assertEquals(ddlQueries4.size(), 6);
}
/**
diff --git
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
index fdd65eb..a07e193 100755
---
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
+++
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
@@ -26,9 +26,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
-import java.util.Collection;
+import java.util.List;
import java.util.Map;
+import static
org.apache.atlas.hive.hook.events.BaseHiveEvent.ATTRIBUTE_DDL_QUERIES;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.testng.AssertJUnit.*;
@@ -42,10 +43,10 @@ public class HiveMetastoreHookIT extends HiveITBase {
String query = "CREATE DATABASE " + dbName;
runCommand(query);
- String dbId = assertDatabaseIsRegistered(dbName);
-
+ String dbId = assertDatabaseIsRegistered(dbName);
AtlasEntity dbEntity = getAtlasEntity(dbId);
- assertNotNull(dbEntity);
+
+ assertEquals(((List)
dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
}
@Test (priority = 2)
@@ -66,6 +67,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
dbEntity = getAtlasEntity(dbId);
Map parameters = (Map) dbEntity.getAttribute("parameters");
+ assertEquals(((List)
dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
assertNotNull(parameters);
assertEquals(2, parameters.size());
@@ -75,6 +77,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
dbEntity = getAtlasEntity(dbId);
+
+ assertEquals(((List)
dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
assertEquals(dbEntity.getAttribute("owner"), "hive");
assertEquals(dbEntity.getAttribute("ownerType"), "USER");
@@ -86,6 +90,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
dbEntity = getAtlasEntity(dbId);
+ assertEquals(((List)
dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
+
String location = (String) dbEntity.getAttribute("location");
assertEquals(location, hdfsPath);
}
@@ -146,9 +152,12 @@ public class HiveMetastoreHookIT extends HiveITBase {
String dbId = assertDatabaseIsRegistered(dbName);
assertEquals(getAtlasEntity(dbId).getStatus(), ACTIVE);
- String tableName = tableName();
+ String tableName = tableName();
runCommand("CREATE TABLE " + dbName + "." + tableName + " (name
string, age int, dob date)");
- String tblId = assertTableIsRegistered(dbName, tableName);
+ String tblId = assertTableIsRegistered(dbName, tableName);
+ AtlasEntity tblEntity = getAtlasEntity(tblId);
+
+ assertEquals(((List)
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
}
@@ -167,8 +176,13 @@ public class HiveMetastoreHookIT extends HiveITBase {
assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
String viewName = tableName();
+
runCommand("CREATE VIEW " + dbName + "." + viewName + " AS SELECT *
FROM " + dbName + "." + tableName);
- tblId = assertTableIsRegistered(dbName, viewName);
+
+ tblId = assertTableIsRegistered(dbName, viewName);
+ AtlasEntity tblEntity = getAtlasEntity(tblId);
+
+ assertEquals(((List)
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
}
@@ -180,11 +194,13 @@ public class HiveMetastoreHookIT extends HiveITBase {
runCommand(query);
String dbId = assertDatabaseIsRegistered(dbName);
assertEquals(getAtlasEntity(dbId).getStatus(), ACTIVE);
+ assertEquals(((List)
getAtlasEntity(dbId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(),
0);
String tableName = tableName();
runCommand("CREATE TABLE " + dbName + "." + tableName + " (name
string, age int, dob date)");
String tblId = assertTableIsRegistered(dbName, tableName);
assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
+ assertEquals(((List)
getAtlasEntity(tblId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(),
0);
// SET TBLPROPERTIES
query = "ALTER TABLE " + dbName + "." + tableName + " SET
TBLPROPERTIES (\"prop1\"=\"val1\", \"prop2\"=\"val2\", \"prop3\"=\"val3\")";
@@ -216,6 +232,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
assertEquals(serdeAttrs.get("serializationLib"),
"org.apache.hadoop.hive.ql.io.orc.OrcSerde");
assertEquals(((Map) serdeAttrs.get("parameters")).get("prop1"),
"val1");
assertEquals(((Map) serdeAttrs.get("parameters")).get("prop2"),
"val2");
+ assertEquals(((List)
tableEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
}
@Test (priority = 8)
@@ -240,6 +257,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
AtlasEntityWithExtInfo tableEntityWithExtInfo =
getAtlasEntityWithExtInfo(tblId);
AtlasEntity tableEntity =
tableEntityWithExtInfo.getEntity();
+ assertEquals(((List)
tableEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
+
// validate table rename in table entity
assertEquals(newTableName, tableEntity.getAttribute("name"));
assertTrue(((String)
tableEntity.getAttribute("qualifiedName")).contains(newTableName));
@@ -284,6 +303,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
AtlasEntity col2Entity = getAtlasEntity(col2Id);
assertEquals(col2Entity.getAttribute("name"), "col22");
assertEquals(col2Entity.getAttribute("type"), "string");
+ assertEquals(((List)
getAtlasEntity(tblId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(),
0);
}
@Test (priority = 10)