This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push: new 2341d58 ATLAS-3035: updated entity-get/delete to retrieve/delete soft-ref entities 2341d58 is described below commit 2341d58a6ef14de410b75abdbc1378c6f81db6fa Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Tue Jan 22 19:39:37 2019 -0800 ATLAS-3035: updated entity-get/delete to retrieve/delete soft-ref entities --- .../org/apache/atlas/utils/AtlasEntityUtil.java | 97 +++++++++++++++++- .../repository/store/graph/v1/DeleteHandlerV1.java | 63 ++++++++++-- .../store/graph/v2/EntityGraphMapper.java | 26 ++--- .../store/graph/v2/EntityGraphRetriever.java | 111 +++++++++++---------- 4 files changed, 214 insertions(+), 83 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java b/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java index 27409c7..316496d 100644 --- a/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java +++ b/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java @@ -19,24 +19,30 @@ package org.apache.atlas.utils; import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.type.AtlasArrayType; +import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasMapType; -import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Objects; public class AtlasEntityUtil { private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityUtil.class); + private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":"; + private static final String SOFT_REF_FORMAT = "%s" + SOFT_REFERENCE_FORMAT_SEPERATOR + "%s"; + private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0; + private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1; + public static boolean hasAnyAttributeUpdate(AtlasEntityType entityType, AtlasEntity currEntity, AtlasEntity newEntity) { if (LOG.isDebugEnabled()) { LOG.debug("==> hasAnyAttributeUpdate(guid={}, typeName={})", currEntity.getGuid(), currEntity.getTypeName()); @@ -68,4 +74,85 @@ public class AtlasEntityUtil { return ret; } + + + public static String formatSoftRefValue(String typeName, String guid) { + return String.format(SOFT_REF_FORMAT, typeName, guid); + } + + public static String formatSoftRefValue(AtlasObjectId objectId) { + return formatSoftRefValue(objectId.getTypeName(), objectId.getGuid()); + } + + public static List<String> formatSoftRefValue(List<AtlasObjectId> objIds) { + List<String> ret = new ArrayList<>(); + + for (AtlasObjectId objId : objIds) { + ret.add(formatSoftRefValue(objId)); + } + + return ret; + } + + public static Map<String, String> formatSoftRefValue(Map<String, AtlasObjectId> objIdMap) { + Map<String, String> ret = new HashMap<>(); + + for (Map.Entry<String, AtlasObjectId> entry : objIdMap.entrySet()) { + ret.put(entry.getKey(), formatSoftRefValue(entry.getValue())); + } + + return ret; + } + + public static AtlasObjectId parseSoftRefValue(String softRefValue) { + AtlasObjectId ret = null; + + if (StringUtils.isNotEmpty(softRefValue)) { + String[] objectIdParts = StringUtils.split(softRefValue, SOFT_REFERENCE_FORMAT_SEPERATOR); + + if(objectIdParts.length >= 2) { + ret = new AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID], objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]); + } else { + LOG.warn("Invalid soft-ref value: {}", softRefValue); + } + } + + return ret; + } + + public static List<AtlasObjectId> parseSoftRefValue(List<String> softRefValue) { + List<AtlasObjectId> ret = null; + + if (CollectionUtils.isNotEmpty(softRefValue)) { + ret = new ArrayList<>(); + + for (String elemValue : softRefValue) { + AtlasObjectId objId = parseSoftRefValue(elemValue); + + if (objId != null) { + ret.add(objId); + } + } + } + + return ret; + } + + public static Map<String, AtlasObjectId> parseSoftRefValue(Map<String, String> softRefValue) { + Map<String, AtlasObjectId> ret = null; + + if (MapUtils.isNotEmpty(softRefValue)) { + ret = new HashMap<>(); + + for (Map.Entry<String, String> entry : softRefValue.entrySet()) { + AtlasObjectId objId = parseSoftRefValue(entry.getValue()); + + if (objId != null) { + ret.put(entry.getKey(), objId); + } + } + } + + return ret; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index c57e30a..8170e1b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -45,6 +45,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; @@ -210,14 +211,23 @@ public abstract class DeleteHandlerV1 { TypeCategory typeCategory = attrType.getTypeCategory(); if (typeCategory == OBJECT_ID_TYPE) { - AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel); + if (attributeInfo.getAttributeDef().isSoftReferenced()) { + String softRefVal = vertex.getProperty(attributeInfo.getVertexPropertyName(), String.class); + AtlasObjectId refObjId = AtlasEntityUtil.parseSoftRefValue(softRefVal); + AtlasVertex refVertex = refObjId != null ? AtlasGraphUtilsV2.findByGuid(refObjId.getGuid()) : null; - if (edge == null || getState(edge) == DELETED) { - continue; - } + if (refVertex != null) { + vertices.push(refVertex); + } + } else { + AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel); - vertices.push(edge.getInVertex()); + if (edge == null || getState(edge) == DELETED) { + continue; + } + vertices.push(edge.getInVertex()); + } } else if (typeCategory == ARRAY || typeCategory == MAP) { TypeCategory elementType = null; @@ -231,15 +241,46 @@ public abstract class DeleteHandlerV1 { continue; } - List<AtlasEdge> edges = getCollectionElementsUsingRelationship(vertex, attributeInfo); + if (attributeInfo.getAttributeDef().isSoftReferenced()) { + if (typeCategory == ARRAY) { + List softRefVal = vertex.getListProperty(attributeInfo.getVertexPropertyName(), List.class); + List<AtlasObjectId> refObjIds = AtlasEntityUtil.parseSoftRefValue(softRefVal); + + if (CollectionUtils.isNotEmpty(refObjIds)) { + for (AtlasObjectId refObjId : refObjIds) { + AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(refObjId.getGuid()); + + if (refVertex != null) { + vertices.push(refVertex); + } + } + } + } else if (typeCategory == MAP) { + Map softRefVal = vertex.getProperty(attributeInfo.getVertexPropertyName(), Map.class); + Map<String, AtlasObjectId> refObjIds = AtlasEntityUtil.parseSoftRefValue(softRefVal); + + if (MapUtils.isNotEmpty(refObjIds)) { + for (AtlasObjectId refObjId : refObjIds.values()) { + AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(refObjId.getGuid()); - if (CollectionUtils.isNotEmpty(edges)) { - for (AtlasEdge edge : edges) { - if (edge == null || getState(edge) == DELETED) { - continue; + if (refVertex != null) { + vertices.push(refVertex); + } + } } + } + + } else { + List<AtlasEdge> edges = getCollectionElementsUsingRelationship(vertex, attributeInfo); - vertices.push(edge.getInVertex()); + if (CollectionUtils.isNotEmpty(edges)) { + for (AtlasEdge edge : edges) { + if (edge == null || getState(edge) == DELETED) { + continue; + } + + vertices.push(edge.getInVertex()); + } } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index afa471f..967b052 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdg import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; import org.apache.commons.codec.digest.DigestUtils; @@ -496,28 +497,27 @@ public class EntityGraphMapper { } } - private Object mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) { - if(ctx.getValue() != null && !(ctx.getValue() instanceof AtlasObjectId)) { - LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found: {}", ctx.getValue().getClass()); - return null; - } + private String mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) { + String ret = null; - String softRefValue = null; - if(ctx.getValue() != null) { + if (ctx.getValue() instanceof AtlasObjectId) { AtlasObjectId objectId = (AtlasObjectId) ctx.getValue(); - String resolvedGuid = AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid()) - ? context.getGuidAssignments().get(objectId.getGuid()) - : objectId.getGuid(); + String typeName = objectId.getTypeName(); + String guid = AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid()) ? context.getGuidAssignments().get(objectId.getGuid()) : objectId.getGuid(); - softRefValue = String.format(SOFT_REF_FORMAT, objectId.getTypeName(), resolvedGuid); + ret = AtlasEntityUtil.formatSoftRefValue(typeName, guid); + } else { + if (ctx.getValue() != null) { + LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found: {}", ctx.getValue().getClass()); + } } - return softRefValue; + return ret; } private Object mapSoftRefValueWithUpdate(AttributeMutationContext ctx, EntityMutationContext context) { + String softRefValue = mapSoftRefValue(ctx, context); - String softRefValue = (String) mapSoftRefValue(ctx, context); AtlasGraphUtilsV2.setProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), softRefValue); return softRefValue; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 79216d5..8f8ddb8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -52,6 +52,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.atlas.utils.AtlasJson; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -71,6 +72,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -106,9 +108,6 @@ public class EntityGraphRetriever { public static final String DESCRIPTION = "description"; public static final String OWNER = "owner"; public static final String CREATE_TIME = "createTime"; - private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":"; - private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0; - private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1; public static final String QUALIFIED_NAME = "qualifiedName"; private static final TypeReference<List<TimeBoundary>> TIME_BOUNDARIES_LIST_TYPE = new TypeReference<List<TimeBoundary>>() {}; @@ -704,21 +703,21 @@ public class EntityGraphRetriever { break; case OBJECT_ID_TYPE: if(attribute.getAttributeDef().isSoftReferenced()) { - ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute.getVertexPropertyName()); + ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo); } else { ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo); } break; case ARRAY: if(attribute.getAttributeDef().isSoftReferenced()) { - ret = mapVertexToArrayForSoftRef(entityVertex, attribute.getVertexPropertyName()); + ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo); } else { ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo); } break; case MAP: if(attribute.getAttributeDef().isSoftReferenced()) { - ret = mapVertexToMapForSoftRef(entityVertex, attribute.getVertexPropertyName()); + ret = mapVertexToMapForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo); } else { ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo); } @@ -731,74 +730,78 @@ public class EntityGraphRetriever { return ret; } - private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex entityVertex, String propertyName) { - Map map = entityVertex.getProperty(propertyName, Map.class); - if (MapUtils.isEmpty(map)) { - return null; - } + private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) { + Map<String, AtlasObjectId> ret = null; + Map softRefVal = entityVertex.getProperty(attribute.getVertexPropertyName(), Map.class); - if (LOG.isDebugEnabled()) { - LOG.debug("Mapping map attribute {} for vertex {}", propertyName, entityVertex); - } + if (MapUtils.isNotEmpty(softRefVal)) { + ret = new HashMap<>(); - Map ret = new HashMap(); - for (Object mapKey : map.keySet()) { - String softRefRaw = (String) map.get(mapKey); - AtlasObjectId mapValue = getAtlasObjectIdFromSoftRefFormat(softRefRaw); - if (mapValue != null) { - ret.put(mapKey, mapValue); + for (Object mapKey : softRefVal.keySet()) { + AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat(Objects.toString(softRefVal.get(mapKey)), attribute, entityExtInfo, isMinExtInfo); + + if (objectId != null) { + ret.put(Objects.toString(mapKey), objectId); + } } } return ret; } - private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex entityVertex, String propertyName) { - List rawValue = entityVertex.getListProperty(propertyName, List.class); - if (CollectionUtils.isEmpty(rawValue)) { - return null; - } + private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) { + List<AtlasObjectId> ret = null; + List softRefVal = entityVertex.getListProperty(attribute.getVertexPropertyName(), List.class); - List list = (List) rawValue; - List<AtlasObjectId> objectIds = new ArrayList<>(); - for (Object o : list) { - if (!(o instanceof String)) { - continue; - } + if (CollectionUtils.isNotEmpty(softRefVal)) { + ret = new ArrayList<>(); - AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat((String) o); - if(objectId == null) { - continue; - } + for (Object o : softRefVal) { + AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat(Objects.toString(o), attribute, entityExtInfo, isMinExtInfo); - objectIds.add(objectId); + if(objectId != null) { + ret.add(objectId); + } + } } - return objectIds; + return ret; } - private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, String vertexPropertyName) { - String rawValue = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, vertexPropertyName, String.class); - if(StringUtils.isEmpty(rawValue)) { - return null; - } + private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) { + String softRefVal = AtlasGraphUtilsV2.getEncodedProperty(entityVertex, attribute.getVertexPropertyName(), String.class); - return getAtlasObjectIdFromSoftRefFormat(rawValue); + return StringUtils.isNotEmpty(softRefVal) ? getAtlasObjectIdFromSoftRefFormat(softRefVal, attribute, entityExtInfo, isMinExtInfo) : null; } - private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String rawValue) { - if(StringUtils.isEmpty(rawValue)) { - return null; - } + private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String softRefVal, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) { + AtlasObjectId ret = AtlasEntityUtil.parseSoftRefValue(softRefVal); - String[] objectIdParts = StringUtils.split(rawValue, SOFT_REFERENCE_FORMAT_SEPERATOR); - if(objectIdParts.length < 2) { - LOG.warn("Expecting value to be formatted for softRef. Instead found: {}", rawValue); - return null; + if(ret != null) { + if (entityExtInfo != null && attribute.isOwnedRef()) { + try { + AtlasVertex referenceVertex = getEntityVertex(ret.getGuid()); + + if (referenceVertex != null) { + final AtlasEntity entity; + + if (isMinExtInfo) { + entity = mapVertexToAtlasEntityMin(referenceVertex, entityExtInfo); + } else { + entity = mapVertexToAtlasEntity(referenceVertex, entityExtInfo); + } + + if (entity != null) { + ret = toAtlasObjectId(entity); + } + } + } catch (AtlasBaseException excp) { + LOG.info("failed to retrieve soft-referenced entity(typeName={}, guid={}); errorCode={}. Ignoring", ret.getTypeName(), ret.getGuid(), excp.getAtlasErrorCode()); + } + } } - return new AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID], - objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]); + return ret; } private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, @@ -986,7 +989,7 @@ public class EntityGraphRetriever { ret = AtlasTypeUtil.getAtlasObjectId(entity); } } else { - ret = new AtlasObjectId(getGuid(referenceVertex), getTypeName(referenceVertex)); + ret = toAtlasObjectId(referenceVertex); } } }