Repository: atlas Updated Branches: refs/heads/master d003ddb33 -> 40329046a
ATLAS-2965: Duplicate entities are created when when same qualifiedName is given but different guids Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/40329046 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/40329046 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/40329046 Branch: refs/heads/master Commit: 40329046a1d67059409c02396d696153bb2de653 Parents: d003ddb Author: Sarath Subramanian <[email protected]> Authored: Thu Nov 15 15:42:19 2018 -0800 Committer: Sarath Subramanian <[email protected]> Committed: Thu Nov 15 15:42:19 2018 -0800 ---------------------------------------------------------------------- .../test/java/org/apache/atlas/TestUtilsV2.java | 33 ++++++++++++++++++++ .../store/graph/v2/AtlasEntityStoreV2.java | 33 +++++++++++++++++++- .../store/graph/v2/EntityGraphRetriever.java | 21 +++++++++++++ .../graph/v2/AtlasComplexAttributesTest.java | 21 ++++++++++--- .../store/graph/v2/AtlasEntityStoreV2Test.java | 21 +++++++++++++ 5 files changed, 124 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/40329046/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index 4e9a651..093105b 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -1256,6 +1256,39 @@ public final class TestUtilsV2 { return newtestEntityDef; } + public static AtlasEntityWithExtInfo createTableEntityDuplicatesV2(AtlasEntity dbEntity) { + AtlasEntity tblEntity = new AtlasEntity(TABLE_TYPE); + + tblEntity.setAttribute(NAME, RandomStringUtils.randomAlphanumeric(10)); + tblEntity.setAttribute("description", "random table"); + tblEntity.setAttribute("type", "type"); + tblEntity.setAttribute("tableType", "MANAGED"); + tblEntity.setAttribute("database", AtlasTypeUtil.getAtlasObjectId(dbEntity)); + + AtlasEntity col1 = createColumnEntity(tblEntity); + col1.setAttribute(NAME, "col1"); + + AtlasEntity col2 = createColumnEntity(tblEntity); + col2.setAttribute(NAME, "col1"); + + AtlasEntity col3 = createColumnEntity(tblEntity); + col3.setAttribute(NAME, "col1"); + + // all 3 columns have different guid but same typeName and unique attributes + tblEntity.setAttribute(COLUMNS_ATTR_NAME, Arrays.asList(AtlasTypeUtil.getAtlasObjectId(col1), + AtlasTypeUtil.getAtlasObjectId(col2), + AtlasTypeUtil.getAtlasObjectId(col3))); + + AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(tblEntity); + + ret.addReferredEntity(dbEntity); + ret.addReferredEntity(col1); + ret.addReferredEntity(col2); + ret.addReferredEntity(col3); + + return ret; + } + public static AtlasEntity createColumnEntity(AtlasEntity tableEntity) { return createColumnEntity(tableEntity, "col" + seq.addAndGet(1)); } http://git-wip-us.apache.org/repos/asf/atlas/blob/40329046/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index ee8de1f..6ee0279 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -742,10 +742,11 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { RequestContext requestContext = RequestContext.get(); for (String guid : discoveryContext.getReferencedGuids()) { - AtlasVertex vertex = discoveryContext.getResolvedEntityVertex(guid); AtlasEntity entity = entityStream.getByGuid(guid); if (entity != null) { // entity would be null if guid is not in the stream but referenced by an entity in the stream + AtlasVertex vertex = getResolvedEntityVertex(discoveryContext, entity); + if (vertex != null) { if (!isPartialUpdate) { graphDiscoverer.validateAndNormalize(entity); @@ -779,6 +780,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { discoveryContext.addResolvedGuid(guid, vertex); + discoveryContext.addResolvedIdByUniqAttribs(getAtlasObjectId(entity), vertex); + String generatedGuid = AtlasGraphUtilsV2.getIdFromVertex(vertex); entity.setGuid(generatedGuid); @@ -798,6 +801,34 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { return context; } + private AtlasVertex getResolvedEntityVertex(EntityGraphDiscoveryContext context, AtlasEntity entity) throws AtlasBaseException { + AtlasObjectId objectId = getAtlasObjectId(entity); + AtlasVertex ret = context.getResolvedEntityVertex(entity.getGuid()); + + if (ret != null) { + context.addResolvedIdByUniqAttribs(objectId, ret); + } else { + ret = context.getResolvedEntityVertex(objectId); + + if (ret != null) { + context.addResolvedGuid(entity.getGuid(), ret); + } + } + + return ret; + } + + private AtlasObjectId getAtlasObjectId(AtlasEntity entity) { + AtlasObjectId ret = entityRetriever.toAtlasObjectId(entity); + + if (ret != null && MapUtils.isNotEmpty(ret.getUniqueAttributes())) { + // if uniqueAttributes is not empty, reset guid to null. + ret.setGuid(null); + } + + return ret; + } + private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCandidates) throws AtlasBaseException { EntityMutationResponse response = new EntityMutationResponse(); RequestContext req = RequestContext.get(); http://git-wip-us.apache.org/repos/asf/atlas/blob/40329046/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index c8a760b..912bc3e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -260,6 +260,27 @@ public final class EntityGraphRetriever { return ret; } + public AtlasObjectId toAtlasObjectId(AtlasEntity entity) { + AtlasObjectId ret = null; + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + if (entityType != null) { + Map<String, Object> uniqueAttributes = new HashMap<>(); + + for (String attributeName : entityType.getUniqAttributes().keySet()) { + Object attrValue = entity.getAttribute(attributeName); + + if (attrValue != null) { + uniqueAttributes.put(attributeName, attrValue); + } + } + + ret = new AtlasObjectId(entity.getGuid(), entity.getTypeName(), uniqueAttributes); + } + + return ret; + } + public AtlasClassification toAtlasClassification(AtlasVertex classificationVertex) throws AtlasBaseException { AtlasClassification ret = new AtlasClassification(getTypeName(classificationVertex)); http://git-wip-us.apache.org/repos/asf/atlas/blob/40329046/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java index ebd5f0f..cf7fa08 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java @@ -30,6 +30,7 @@ import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.type.AtlasEntityType; import org.apache.commons.lang.time.DateUtils; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; @@ -256,8 +257,9 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { init(); AtlasEntity complexEntity = getEntityFromStore(complexCollectionAttrEntity.getEntity().getGuid()); AtlasEntitiesWithExtInfo complexEntitiesInfo = new AtlasEntitiesWithExtInfo(complexEntity); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(ENTITY_TYPE); - // Modify array of entities + // Replace list of entities with new values AtlasEntity e0Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray00"); put("isReplicated", true); }}); AtlasEntity e1Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray11"); put("isReplicated", false); }}); AtlasEntity e2Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray22"); put("isReplicated", true); }}); @@ -274,11 +276,18 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { AtlasEntityHeader updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR); validateEntity(complexEntitiesInfo, getEntityFromStore(updatedComplexEntity)); - // add a new element to array of entities + // add a new element to list of entities init(); + + e0Array = entityStore.getByUniqueAttributes(entityType, new HashMap<String, Object>() {{ put(NAME, "entityArray00"); put("isReplicated", true); }}).getEntity(); + e1Array = entityStore.getByUniqueAttributes(entityType, new HashMap<String, Object>() {{ put(NAME, "entityArray11"); put("isReplicated", false); }}).getEntity(); + e2Array = entityStore.getByUniqueAttributes(entityType, new HashMap<String, Object>() {{ put(NAME, "entityArray22"); put("isReplicated", true); }}).getEntity(); AtlasEntity e3Array = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray33"); put("isReplicated", true); }}); - entityList.add(getAtlasObjectId(e3Array)); + + entityList = new ArrayList<>(Arrays.asList(getAtlasObjectId(e0Array), getAtlasObjectId(e1Array), getAtlasObjectId(e2Array), getAtlasObjectId(e3Array))); + complexEntity.setAttribute("listOfEntities", entityList); + complexEntitiesInfo.getReferredEntities().clear(); complexEntitiesInfo.addReferredEntity(e3Array); response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false); @@ -287,8 +296,10 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { // remove one of the entity values - entityArray00 init(); - entityList.remove(0); + e3Array = entityStore.getByUniqueAttributes(entityType, new HashMap<String, Object>() {{ put(NAME, "entityArray33"); put("isReplicated", true); }}).getEntity(); + entityList = new ArrayList<>(Arrays.asList(getAtlasObjectId(e1Array), getAtlasObjectId(e2Array), getAtlasObjectId(e3Array))); complexEntity.setAttribute("listOfEntities", entityList); + complexEntitiesInfo.getReferredEntities().clear(); response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false); updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR); @@ -308,6 +319,7 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { AtlasEntity e3Array_duplicate = new AtlasEntity(ENTITY_TYPE, new HashMap<String, Object>() {{ put(NAME, "entityArray33"); put("isReplicated", true); }}); entityList.add(getAtlasObjectId(e3Array_duplicate)); complexEntity.setAttribute("listOfEntities", entityList); + complexEntitiesInfo.getReferredEntities().clear(); complexEntitiesInfo.addReferredEntity(e3Array_duplicate); response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false); @@ -318,6 +330,7 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { init(); entityList.clear(); complexEntity.setAttribute("listOfEntities", entityList); + complexEntitiesInfo.getReferredEntities().clear(); response = entityStore.createOrUpdate(new AtlasEntityStream(complexEntitiesInfo), false); updatedComplexEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR); http://git-wip-us.apache.org/repos/asf/atlas/blob/40329046/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java index ca46ffd..d49f4a5 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java @@ -923,6 +923,27 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase { } } + @Test + public void testCreateWithDuplicateGuids() throws Exception { + init(); + AtlasEntityWithExtInfo tblEntity2 = TestUtilsV2.createTableEntityDuplicatesV2(dbEntity.getEntity()); + EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(tblEntity2), false); + + List<AtlasEntityHeader> createdEntities = response.getCreatedEntities(); + assertTrue(CollectionUtils.isNotEmpty(createdEntities)); + assertEquals(createdEntities.size(), 2); + + String tableGuid = createdEntities.get(0).getGuid(); + AtlasEntityWithExtInfo tableEntity = entityStore.getById(tableGuid); + assertEquals(tableEntity.getReferredEntities().size(), 1); + + List<AtlasObjectId> columns = (List<AtlasObjectId>) tableEntity.getEntity().getAttribute("columns"); + assertEquals(columns.size(), 1); + + Set<AtlasObjectId> uniqueColumns = new HashSet<>(columns); + assertTrue(uniqueColumns.size() == 1); + } + @Test(dependsOnMethods = "testCreate") public void associateSameTagToMultipleEntities() throws AtlasBaseException { final String TAG_NAME = "tagx";
