Repository: atlas Updated Branches: refs/heads/branch-0.8 110db1a40 -> 16456800b
ATLAS-2878: avoid retrieval of entiyWithExtInfo when extInfo is not needed Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/16456800 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/16456800 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/16456800 Branch: refs/heads/branch-0.8 Commit: 16456800b2c11969cc17de3a6423b955a958a5b2 Parents: 110db1a Author: Madhan Neethiraj <mad...@apache.org> Authored: Tue Sep 18 12:56:08 2018 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Tue Sep 18 15:15:39 2018 -0700 ---------------------------------------------------------------------- .../repository/graph/FullTextMapperV2.java | 45 ++++++++++++++---- .../java/org/apache/atlas/RequestContext.java | 49 +++++++++++++------- .../NotificationHookConsumerTest.java | 3 +- 3 files changed, 69 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/16456800/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java index 76acf8c..5b5158c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java @@ -73,8 +73,16 @@ public class FullTextMapperV2 { * @throws AtlasBaseException */ public String getIndexTextForClassifications(String guid, List<AtlasClassification> classifications) throws AtlasBaseException { - String ret = null; - AtlasEntityWithExtInfo entityWithExtInfo = getAndCacheEntity(guid); + String ret = null; + final AtlasEntityWithExtInfo entityWithExtInfo; + + if (followReferences) { + entityWithExtInfo = getAndCacheEntityWithExtInfo(guid); + } else { + AtlasEntity entity = getAndCacheEntity(guid); + + entityWithExtInfo = entity != null ? new AtlasEntityWithExtInfo(entity) : null; + } if (entityWithExtInfo != null) { StringBuilder sb = new StringBuilder(); @@ -100,13 +108,13 @@ public class FullTextMapperV2 { } public String getIndexTextForEntity(String guid) throws AtlasBaseException { - String ret = null; - AtlasEntityWithExtInfo entity = getAndCacheEntity(guid); + String ret = null; + AtlasEntity entity = getAndCacheEntity(guid); if (entity != null) { StringBuilder sb = new StringBuilder(); - map(entity.getEntity(), entity, sb, new HashSet<String>()); + map(entity, null, sb, new HashSet<String>()); ret = sb.toString(); } @@ -165,7 +173,7 @@ public class FullTextMapperV2 { private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException { if (value instanceof AtlasObjectId) { - if (followReferences) { + if (followReferences && entityExtInfo != null) { AtlasObjectId objectId = (AtlasObjectId) value; AtlasEntity entity = entityExtInfo.getEntity(objectId.getGuid()); @@ -202,9 +210,28 @@ public class FullTextMapperV2 { } } - private AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException { - RequestContext context = RequestContext.get(); - AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid); + private AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException { + RequestContext context = RequestContext.get(); + AtlasEntity entity = context.getEntity(guid); + + if (entity == null) { + entity = entityGraphRetriever.toAtlasEntity(guid); + + if (entity != null) { + context.cache(entity); + + if (LOG.isDebugEnabled()) { + LOG.debug("Cache miss -> GUID = {}", guid); + } + } + } + + return entity; + } + + private AtlasEntityWithExtInfo getAndCacheEntityWithExtInfo(String guid) throws AtlasBaseException { + RequestContext context = RequestContext.get(); + AtlasEntityWithExtInfo entityWithExtInfo = context.getEntityWithExtInfo(guid); if (entityWithExtInfo == null) { entityWithExtInfo = entityGraphRetriever.toAtlasEntityWithExtInfo(guid); http://git-wip-us.apache.org/repos/asf/atlas/blob/16456800/server-api/src/main/java/org/apache/atlas/RequestContext.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index f810dc9..6775f4d 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Set; import org.apache.atlas.metrics.Metrics; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.slf4j.Logger; @@ -40,14 +41,15 @@ public class RequestContext { private static final ThreadLocal<RequestContext> CURRENT_CONTEXT = new ThreadLocal<>(); private static final Set<RequestContext> ACTIVE_REQUESTS = new HashSet<>(); - private final Set<String> createdEntityIds = new LinkedHashSet<>(); - private final Set<String> updatedEntityIds = new LinkedHashSet<>(); - private final Set<String> deletedEntityIds = new LinkedHashSet<>(); - private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>(); - private final Map<String,ITypedReferenceableInstance> entityCacheV1 = new HashMap<>(); - private final Map<String,AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>(); - private final Metrics metrics = new Metrics(); - private final long requestTime = System.currentTimeMillis(); + private final Set<String> createdEntityIds = new LinkedHashSet<>(); + private final Set<String> updatedEntityIds = new LinkedHashSet<>(); + private final Set<String> deletedEntityIds = new LinkedHashSet<>(); + private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>(); + private final Map<String,ITypedReferenceableInstance> entityCacheV1 = new HashMap<>(); + private final Map<String,AtlasEntity> entityCache = new HashMap<>(); + private final Map<String,AtlasEntityWithExtInfo> entityExtInfoCache = new HashMap<>(); + private final Metrics metrics = new Metrics(); + private final long requestTime = System.currentTimeMillis(); private String user; private int maxAttempts = 1; @@ -79,13 +81,13 @@ public class RequestContext { RequestContext instance = CURRENT_CONTEXT.get(); if (instance != null) { - if (instance.entityCacheV1 != null) { - instance.entityCacheV1.clear(); - } - - if (instance.entityCacheV2 != null) { - instance.entityCacheV2.clear(); - } + instance.createdEntityIds.clear(); + instance.updatedEntityIds.clear(); + instance.deletedEntityIds.clear(); + instance.deletedEntities.clear(); + instance.entityCacheV1.clear(); + instance.entityCache.clear(); + instance.entityExtInfoCache.clear(); synchronized (ACTIVE_REQUESTS) { ACTIVE_REQUESTS.remove(instance); @@ -131,9 +133,16 @@ public class RequestContext { * Adds the specified instance to the cache * */ + public void cache(AtlasEntity entity) { + if (entity != null && entity.getGuid() != null) { + entityCache.put(entity.getGuid(), entity); + } + } + public void cache(AtlasEntityWithExtInfo entity) { if (entity != null && entity.getEntity() != null && entity.getEntity().getGuid() != null) { - entityCacheV2.put(entity.getEntity().getGuid(), entity); + entityExtInfoCache.put(entity.getEntity().getGuid(), entity); + entityCache.put(entity.getEntity().getGuid(), entity.getEntity()); } } @@ -155,8 +164,12 @@ public class RequestContext { * @param guid the guid to find * @return Either the instance or null if it is not in the cache. */ - public AtlasEntityWithExtInfo getInstanceV2(String guid) { - return entityCacheV2.get(guid); + public AtlasEntity getEntity(String guid) { + return entityCache.get(guid); + } + + public AtlasEntityWithExtInfo getEntityWithExtInfo(String guid) { + return entityExtInfoCache.get(guid); } public String getUser() { http://git-wip-us.apache.org/repos/asf/atlas/blob/16456800/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java index e2d1022..b63bedd 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java +++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerTest.java @@ -31,6 +31,7 @@ import org.apache.atlas.repository.store.graph.v1.EntityStream; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.web.service.ServiceState; import org.apache.commons.configuration.Configuration; import org.apache.kafka.common.TopicPartition; @@ -141,7 +142,7 @@ public class NotificationHookConsumerTest { HookNotification.EntityCreateRequest message = new HookNotification.EntityCreateRequest("user", new ArrayList<Referenceable>() { { - add(mock(Referenceable.class)); + add(new Referenceable("testType")); } }); when(atlasEntityStore.createOrUpdate(any(EntityStream.class), anyBoolean())).thenThrow(new RuntimeException("Simulating exception in processing message"));
