Repository: atlas Updated Branches: refs/heads/branch-1.0 a00875362 -> be96cd4aa
ATLAS-2878: avoid retrieval of entiyWithExtInfo when extInfo is not needed Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/be96cd4a Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/be96cd4a Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/be96cd4a Branch: refs/heads/branch-1.0 Commit: be96cd4aa327746d4c51cafb9653225f0fe6d15c Parents: a008753 Author: Madhan Neethiraj <mad...@apache.org> Authored: Tue Sep 18 11:16:50 2018 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Tue Sep 18 14:08:01 2018 -0700 ---------------------------------------------------------------------- .../repository/audit/EntityAuditListenerV2.java | 6 +-- .../converters/AtlasInstanceConverter.java | 30 ++++++++++--- .../repository/graph/FullTextMapperV2.java | 45 ++++++++++++++++---- .../graph/v2/AtlasEntityChangeNotifier.java | 9 ++-- .../store/graph/v2/EntityGraphMapper.java | 14 +++--- .../java/org/apache/atlas/RequestContext.java | 24 ++++++++--- 6 files changed, 90 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/be96cd4a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java index 91e1f63..8ca8c9a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java @@ -180,8 +180,7 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 { List<EntityAuditEventV2> events = new ArrayList<>(); for (AtlasRelatedObjectId relatedObjectId : entities) { - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid()); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid()); if (entity != null) { events.add(createEvent(entity, TERM_ADD, "Added term: " + term.toAuditString())); @@ -198,8 +197,7 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 { List<EntityAuditEventV2> events = new ArrayList<>(); for (AtlasRelatedObjectId relatedObjectId : entities) { - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid()); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid()); if (entity != null) { events.add(createEvent(entity, TERM_DELETE, "Deleted term: " + term.toAuditString())); http://git-wip-us.apache.org/repos/asf/atlas/blob/be96cd4a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java index b26b353..1f0cc86 100644 --- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java +++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java @@ -28,6 +28,7 @@ import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations.EntityOperation; @@ -94,12 +95,12 @@ public class AtlasInstanceConverter { } public Referenceable getReferenceable(String guid) throws AtlasBaseException { - AtlasEntity.AtlasEntityWithExtInfo entity = getAndCacheEntity(guid); + AtlasEntityWithExtInfo entity = getAndCacheEntityExtInfo(guid); return getReferenceable(entity); } - public Referenceable getReferenceable(AtlasEntity.AtlasEntityWithExtInfo entity) throws AtlasBaseException { + public Referenceable getReferenceable(AtlasEntityWithExtInfo entity) throws AtlasBaseException { AtlasFormatConverter.ConverterContext ctx = new AtlasFormatConverter.ConverterContext(); ctx.addEntity(entity.getEntity()); @@ -291,10 +292,29 @@ public class AtlasInstanceConverter { return ret; } + public AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException { + RequestContext context = RequestContext.get(); + AtlasEntity entity = context.getEntity(guid); - public AtlasEntity.AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException { - RequestContext context = RequestContext.get(); - AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid); + if (entity == null) { + entity = entityGraphRetriever.toAtlasEntity(guid); + + if (entity != null) { + context.cache(entity); + + if (LOG.isDebugEnabled()) { + LOG.debug("Cache miss -> GUID = {}", guid); + } + } + } + + return entity; + } + + + public AtlasEntityWithExtInfo getAndCacheEntityExtInfo(String guid) throws AtlasBaseException { + RequestContext context = RequestContext.get(); + AtlasEntityWithExtInfo entityWithExtInfo = context.getEntityWithExtInfo(guid); if (entityWithExtInfo == null) { entityWithExtInfo = entityGraphRetriever.toAtlasEntityWithExtInfo(guid); http://git-wip-us.apache.org/repos/asf/atlas/blob/be96cd4a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java index bc5ae5c..3fd0b02 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java @@ -73,8 +73,16 @@ public class FullTextMapperV2 { * @throws AtlasBaseException */ public String getIndexTextForClassifications(String guid, List<AtlasClassification> classifications) throws AtlasBaseException { - String ret = null; - AtlasEntityWithExtInfo entityWithExtInfo = getAndCacheEntity(guid); + String ret = null; + final AtlasEntityWithExtInfo entityWithExtInfo; + + if (followReferences) { + entityWithExtInfo = getAndCacheEntityWithExtInfo(guid); + } else { + AtlasEntity entity = getAndCacheEntity(guid); + + entityWithExtInfo = entity != null ? new AtlasEntityWithExtInfo(entity) : null; + } if (entityWithExtInfo != null) { StringBuilder sb = new StringBuilder(); @@ -100,13 +108,13 @@ public class FullTextMapperV2 { } public String getIndexTextForEntity(String guid) throws AtlasBaseException { - String ret = null; - AtlasEntityWithExtInfo entity = getAndCacheEntity(guid); + String ret = null; + AtlasEntity entity = getAndCacheEntity(guid); if (entity != null) { StringBuilder sb = new StringBuilder(); - map(entity.getEntity(), entity, sb, new HashSet<String>()); + map(entity, null, sb, new HashSet<String>()); ret = sb.toString(); } @@ -165,7 +173,7 @@ public class FullTextMapperV2 { private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException { if (value instanceof AtlasObjectId) { - if (followReferences) { + if (followReferences && entityExtInfo != null) { AtlasObjectId objectId = (AtlasObjectId) value; AtlasEntity entity = entityExtInfo.getEntity(objectId.getGuid()); @@ -202,9 +210,28 @@ public class FullTextMapperV2 { } } - private AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException { + private AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException { + RequestContext context = RequestContext.get(); + AtlasEntity entity = context.getEntity(guid); + + if (entity == null) { + entity = entityGraphRetriever.toAtlasEntity(guid); + + if (entity != null) { + context.cache(entity); + + if (LOG.isDebugEnabled()) { + LOG.debug("Cache miss -> GUID = {}", guid); + } + } + } + + return entity; + } + + private AtlasEntityWithExtInfo getAndCacheEntityWithExtInfo(String guid) throws AtlasBaseException { RequestContext context = RequestContext.get(); - AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid); + AtlasEntityWithExtInfo entityWithExtInfo = context.getEntityWithExtInfo(guid); if (entityWithExtInfo == null) { entityWithExtInfo = entityGraphRetriever.toAtlasEntityWithExtInfo(guid); @@ -243,4 +270,4 @@ public class FullTextMapperV2 { return ret; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/atlas/blob/be96cd4a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java index 9d8afdf..a8c3363 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java @@ -254,8 +254,7 @@ public class AtlasEntityChangeNotifier { continue; } - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(guid); - AtlasEntity entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(guid); if (entity == null) { continue; @@ -413,9 +412,7 @@ public class AtlasEntityChangeNotifier { entity.setGuid(entityGuid); } else { - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); - - entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + entity = instanceConverter.getAndCacheEntity(entityGuid); } if (entity != null) { @@ -497,4 +494,4 @@ public class AtlasEntityChangeNotifier { doFullTextMapping(Collections.singletonList(entityHeader)); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/atlas/blob/be96cd4a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index d0c75db..f4b3da3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -1456,8 +1456,7 @@ public class EntityGraphMapper { for (AtlasVertex vertex : notificationVertices) { String entityGuid = GraphHelper.getGuid(vertex); - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex); if (CollectionUtils.isNotEmpty(addedClassifications)) { @@ -1563,8 +1562,7 @@ public class EntityGraphMapper { for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry : removedClassifications.entrySet()) { String guid = GraphHelper.getGuid(entry.getKey()); List<AtlasClassification> deletedClassificationNames = entry.getValue(); - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(guid); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(guid); entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassificationNames); } @@ -1720,9 +1718,8 @@ public class EntityGraphMapper { } for (AtlasVertex vertex : notificationVertices) { - String entityGuid = GraphHelper.getGuid(vertex); - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + String entityGuid = GraphHelper.getGuid(vertex); + AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); if (isActive(entity)) { entityChangeNotifier.onClassificationUpdatedToEntity(entity, updatedClassifications); @@ -1734,8 +1731,7 @@ public class EntityGraphMapper { AtlasVertex vertex = entry.getKey(); List<AtlasClassification> removedClassifications = entry.getValue(); String entityGuid = GraphHelper.getGuid(vertex); - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); if (isActive(entity)) { entityChangeNotifier.onClassificationDeletedFromEntity(entity, removedClassifications); http://git-wip-us.apache.org/repos/asf/atlas/blob/be96cd4a/server-api/src/main/java/org/apache/atlas/RequestContext.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index e23eaec..8fe10fe 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -35,7 +35,8 @@ public class RequestContext { private final Map<String, AtlasObjectId> updatedEntities = new HashMap<>(); private final Map<String, AtlasObjectId> deletedEntities = new HashMap<>(); - private final Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>(); + private final Map<String, AtlasEntity> entityCache = new HashMap<>(); + private final Map<String, AtlasEntityWithExtInfo> entityExtInfoCache = new HashMap<>(); private final Map<String, List<AtlasClassification>> addedPropagations = new HashMap<>(); private final Map<String, List<AtlasClassification>> removedPropagations = new HashMap<>(); private final long requestTime = System.currentTimeMillis(); @@ -70,7 +71,8 @@ public class RequestContext { if (instance != null) { instance.updatedEntities.clear(); instance.deletedEntities.clear(); - instance.entityCacheV2.clear(); + instance.entityCache.clear(); + instance.entityExtInfoCache.clear(); instance.addedPropagations.clear(); instance.removedPropagations.clear(); @@ -174,10 +176,18 @@ public class RequestContext { */ public void cache(AtlasEntityWithExtInfo entity) { if (entity != null && entity.getEntity() != null && entity.getEntity().getGuid() != null) { - entityCacheV2.put(entity.getEntity().getGuid(), entity); + entityExtInfoCache.put(entity.getEntity().getGuid(), entity); + entityCache.put(entity.getEntity().getGuid(), entity.getEntity()); } } + public void cache(AtlasEntity entity) { + if (entity != null && entity.getGuid() != null) { + entityCache.put(entity.getGuid(), entity); + } + } + + public Collection<AtlasObjectId> getUpdatedEntities() { return updatedEntities.values(); } @@ -193,8 +203,12 @@ public class RequestContext { * @param guid the guid to find * @return Either the instance or null if it is not in the cache. */ - public AtlasEntityWithExtInfo getInstanceV2(String guid) { - return entityCacheV2.get(guid); + public AtlasEntityWithExtInfo getEntityWithExtInfo(String guid) { + return entityExtInfoCache.get(guid); + } + + public AtlasEntity getEntity(String guid) { + return entityCache.get(guid); } public long getRequestTime() {