Repository: atlas Updated Branches: refs/heads/master 48e522497 -> a064e0924
ATLAS-2878: avoid retrieval of entiyWithExtInfo when extInfo is not needed (cherry picked from commit 8e7ecf72f32ef6ca282a314d85761742e229a48b) Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a064e092 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a064e092 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a064e092 Branch: refs/heads/master Commit: a064e09242a0b1259d7b1227913786d2fb98f695 Parents: 48e5224 Author: Madhan Neethiraj <mad...@apache.org> Authored: Tue Sep 18 11:16:50 2018 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Tue Sep 18 11:35:24 2018 -0700 ---------------------------------------------------------------------- .../repository/audit/EntityAuditListenerV2.java | 6 +-- .../converters/AtlasInstanceConverter.java | 30 ++++++++++--- .../repository/graph/FullTextMapperV2.java | 45 ++++++++++++++++---- .../graph/v2/AtlasEntityChangeNotifier.java | 9 ++-- .../store/graph/v2/EntityGraphMapper.java | 14 +++--- .../java/org/apache/atlas/RequestContext.java | 24 ++++++++--- 6 files changed, 90 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java index 91e1f63..8ca8c9a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java @@ -180,8 +180,7 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 { List<EntityAuditEventV2> events = new ArrayList<>(); for (AtlasRelatedObjectId relatedObjectId : entities) { - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid()); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid()); if (entity != null) { events.add(createEvent(entity, TERM_ADD, "Added term: " + term.toAuditString())); @@ -198,8 +197,7 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 { List<EntityAuditEventV2> events = new ArrayList<>(); for (AtlasRelatedObjectId relatedObjectId : entities) { - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid()); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid()); if (entity != null) { events.add(createEvent(entity, TERM_DELETE, "Deleted term: " + term.toAuditString())); http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java index b26b353..1f0cc86 100644 --- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java +++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java @@ -28,6 +28,7 @@ import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations.EntityOperation; @@ -94,12 +95,12 @@ public class AtlasInstanceConverter { } public Referenceable getReferenceable(String guid) throws AtlasBaseException { - AtlasEntity.AtlasEntityWithExtInfo entity = getAndCacheEntity(guid); + AtlasEntityWithExtInfo entity = getAndCacheEntityExtInfo(guid); return getReferenceable(entity); } - public Referenceable getReferenceable(AtlasEntity.AtlasEntityWithExtInfo entity) throws AtlasBaseException { + public Referenceable getReferenceable(AtlasEntityWithExtInfo entity) throws AtlasBaseException { AtlasFormatConverter.ConverterContext ctx = new AtlasFormatConverter.ConverterContext(); ctx.addEntity(entity.getEntity()); @@ -291,10 +292,29 @@ public class AtlasInstanceConverter { return ret; } + public AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException { + RequestContext context = RequestContext.get(); + AtlasEntity entity = context.getEntity(guid); - public AtlasEntity.AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException { - RequestContext context = RequestContext.get(); - AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid); + if (entity == null) { + entity = entityGraphRetriever.toAtlasEntity(guid); + + if (entity != null) { + context.cache(entity); + + if (LOG.isDebugEnabled()) { + LOG.debug("Cache miss -> GUID = {}", guid); + } + } + } + + return entity; + } + + + public AtlasEntityWithExtInfo getAndCacheEntityExtInfo(String guid) throws AtlasBaseException { + RequestContext context = RequestContext.get(); + AtlasEntityWithExtInfo entityWithExtInfo = context.getEntityWithExtInfo(guid); if (entityWithExtInfo == null) { entityWithExtInfo = entityGraphRetriever.toAtlasEntityWithExtInfo(guid); http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java index b85dc9b..08ccd9c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java @@ -74,8 +74,16 @@ public class FullTextMapperV2 { * @throws AtlasBaseException */ public String getIndexTextForClassifications(String guid, List<AtlasClassification> classifications) throws AtlasBaseException { - String ret = null; - AtlasEntityWithExtInfo entityWithExtInfo = getAndCacheEntity(guid); + String ret = null; + final AtlasEntityWithExtInfo entityWithExtInfo; + + if (followReferences) { + entityWithExtInfo = getAndCacheEntityWithExtInfo(guid); + } else { + AtlasEntity entity = getAndCacheEntity(guid); + + entityWithExtInfo = entity != null ? new AtlasEntityWithExtInfo(entity) : null; + } if (entityWithExtInfo != null) { StringBuilder sb = new StringBuilder(); @@ -101,13 +109,13 @@ public class FullTextMapperV2 { } public String getIndexTextForEntity(String guid) throws AtlasBaseException { - String ret = null; - AtlasEntityWithExtInfo entity = getAndCacheEntity(guid); + String ret = null; + AtlasEntity entity = getAndCacheEntity(guid); if (entity != null) { StringBuilder sb = new StringBuilder(); - map(entity.getEntity(), entity, sb, new HashSet<String>()); + map(entity, null, sb, new HashSet<String>()); ret = sb.toString(); } @@ -166,7 +174,7 @@ public class FullTextMapperV2 { private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException { if (value instanceof AtlasObjectId) { - if (followReferences) { + if (followReferences && entityExtInfo != null) { AtlasObjectId objectId = (AtlasObjectId) value; AtlasEntity entity = entityExtInfo.getEntity(objectId.getGuid()); @@ -203,9 +211,28 @@ public class FullTextMapperV2 { } } - private AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException { + private AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException { + RequestContext context = RequestContext.get(); + AtlasEntity entity = context.getEntity(guid); + + if (entity == null) { + entity = entityGraphRetriever.toAtlasEntity(guid); + + if (entity != null) { + context.cache(entity); + + if (LOG.isDebugEnabled()) { + LOG.debug("Cache miss -> GUID = {}", guid); + } + } + } + + return entity; + } + + private AtlasEntityWithExtInfo getAndCacheEntityWithExtInfo(String guid) throws AtlasBaseException { RequestContext context = RequestContext.get(); - AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid); + AtlasEntityWithExtInfo entityWithExtInfo = context.getEntityWithExtInfo(guid); if (entityWithExtInfo == null) { // Only map ownedRef and relationship attr when follow references is set to true @@ -245,4 +272,4 @@ public class FullTextMapperV2 { return ret; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java index 9d8afdf..a8c3363 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java @@ -254,8 +254,7 @@ public class AtlasEntityChangeNotifier { continue; } - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(guid); - AtlasEntity entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(guid); if (entity == null) { continue; @@ -413,9 +412,7 @@ public class AtlasEntityChangeNotifier { entity.setGuid(entityGuid); } else { - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); - - entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + entity = instanceConverter.getAndCacheEntity(entityGuid); } if (entity != null) { @@ -497,4 +494,4 @@ public class AtlasEntityChangeNotifier { doFullTextMapping(Collections.singletonList(entityHeader)); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index d1e95f0..fa69fdc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -1422,8 +1422,7 @@ public class EntityGraphMapper { for (AtlasVertex vertex : notificationVertices) { String entityGuid = GraphHelper.getGuid(vertex); - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex); if (CollectionUtils.isNotEmpty(addedClassifications)) { @@ -1529,8 +1528,7 @@ public class EntityGraphMapper { for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry : removedClassifications.entrySet()) { String guid = GraphHelper.getGuid(entry.getKey()); List<AtlasClassification> deletedClassificationNames = entry.getValue(); - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(guid); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(guid); entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassificationNames); } @@ -1686,9 +1684,8 @@ public class EntityGraphMapper { } for (AtlasVertex vertex : notificationVertices) { - String entityGuid = GraphHelper.getGuid(vertex); - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + String entityGuid = GraphHelper.getGuid(vertex); + AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); if (isActive(entity)) { entityChangeNotifier.onClassificationUpdatedToEntity(entity, updatedClassifications); @@ -1700,8 +1697,7 @@ public class EntityGraphMapper { AtlasVertex vertex = entry.getKey(); List<AtlasClassification> removedClassifications = entry.getValue(); String entityGuid = GraphHelper.getGuid(vertex); - AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); - AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null; + AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); if (isActive(entity)) { entityChangeNotifier.onClassificationDeletedFromEntity(entity, removedClassifications); http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/server-api/src/main/java/org/apache/atlas/RequestContext.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index e23eaec..8fe10fe 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -35,7 +35,8 @@ public class RequestContext { private final Map<String, AtlasObjectId> updatedEntities = new HashMap<>(); private final Map<String, AtlasObjectId> deletedEntities = new HashMap<>(); - private final Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>(); + private final Map<String, AtlasEntity> entityCache = new HashMap<>(); + private final Map<String, AtlasEntityWithExtInfo> entityExtInfoCache = new HashMap<>(); private final Map<String, List<AtlasClassification>> addedPropagations = new HashMap<>(); private final Map<String, List<AtlasClassification>> removedPropagations = new HashMap<>(); private final long requestTime = System.currentTimeMillis(); @@ -70,7 +71,8 @@ public class RequestContext { if (instance != null) { instance.updatedEntities.clear(); instance.deletedEntities.clear(); - instance.entityCacheV2.clear(); + instance.entityCache.clear(); + instance.entityExtInfoCache.clear(); instance.addedPropagations.clear(); instance.removedPropagations.clear(); @@ -174,10 +176,18 @@ public class RequestContext { */ public void cache(AtlasEntityWithExtInfo entity) { if (entity != null && entity.getEntity() != null && entity.getEntity().getGuid() != null) { - entityCacheV2.put(entity.getEntity().getGuid(), entity); + entityExtInfoCache.put(entity.getEntity().getGuid(), entity); + entityCache.put(entity.getEntity().getGuid(), entity.getEntity()); } } + public void cache(AtlasEntity entity) { + if (entity != null && entity.getGuid() != null) { + entityCache.put(entity.getGuid(), entity); + } + } + + public Collection<AtlasObjectId> getUpdatedEntities() { return updatedEntities.values(); } @@ -193,8 +203,12 @@ public class RequestContext { * @param guid the guid to find * @return Either the instance or null if it is not in the cache. */ - public AtlasEntityWithExtInfo getInstanceV2(String guid) { - return entityCacheV2.get(guid); + public AtlasEntityWithExtInfo getEntityWithExtInfo(String guid) { + return entityExtInfoCache.get(guid); + } + + public AtlasEntity getEntity(String guid) { + return entityCache.get(guid); } public long getRequestTime() {