This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new c40c906 ATLAS-3035: updated entity-get/delete to retrieve/delete
soft-ref entities
c40c906 is described below
commit c40c906da96750a977ebeae39b75123e02b09242
Author: Madhan Neethiraj <mad...@apache.org>
AuthorDate: Tue Jan 22 19:39:37 2019 -0800
ATLAS-3035: updated entity-get/delete to retrieve/delete soft-ref entities
---
.../org/apache/atlas/utils/AtlasEntityUtil.java | 97 +++++++++++++++++-
.../repository/store/graph/v1/DeleteHandlerV1.java | 63 ++++++++++--
.../store/graph/v2/EntityGraphMapper.java | 26 ++---
.../store/graph/v2/EntityGraphRetriever.java | 111 +++++++++++----------
4 files changed, 214 insertions(+), 83 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
b/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
index 27409c7..316496d 100644
--- a/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
+++ b/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
@@ -19,24 +19,30 @@ package org.apache.atlas.utils;
import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasMapType;
-import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import java.util.Objects;
public class AtlasEntityUtil {
private static final Logger LOG =
LoggerFactory.getLogger(AtlasEntityUtil.class);
+ private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":";
+ private static final String SOFT_REF_FORMAT = "%s" +
SOFT_REFERENCE_FORMAT_SEPERATOR + "%s";
+ private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
+ private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1;
+
public static boolean hasAnyAttributeUpdate(AtlasEntityType entityType,
AtlasEntity currEntity, AtlasEntity newEntity) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> hasAnyAttributeUpdate(guid={}, typeName={})",
currEntity.getGuid(), currEntity.getTypeName());
@@ -68,4 +74,85 @@ public class AtlasEntityUtil {
return ret;
}
+
+
+ public static String formatSoftRefValue(String typeName, String guid) {
+ return String.format(SOFT_REF_FORMAT, typeName, guid);
+ }
+
+ public static String formatSoftRefValue(AtlasObjectId objectId) {
+ return formatSoftRefValue(objectId.getTypeName(), objectId.getGuid());
+ }
+
+ public static List<String> formatSoftRefValue(List<AtlasObjectId> objIds) {
+ List<String> ret = new ArrayList<>();
+
+ for (AtlasObjectId objId : objIds) {
+ ret.add(formatSoftRefValue(objId));
+ }
+
+ return ret;
+ }
+
+ public static Map<String, String> formatSoftRefValue(Map<String,
AtlasObjectId> objIdMap) {
+ Map<String, String> ret = new HashMap<>();
+
+ for (Map.Entry<String, AtlasObjectId> entry : objIdMap.entrySet()) {
+ ret.put(entry.getKey(), formatSoftRefValue(entry.getValue()));
+ }
+
+ return ret;
+ }
+
+ public static AtlasObjectId parseSoftRefValue(String softRefValue) {
+ AtlasObjectId ret = null;
+
+ if (StringUtils.isNotEmpty(softRefValue)) {
+ String[] objectIdParts = StringUtils.split(softRefValue,
SOFT_REFERENCE_FORMAT_SEPERATOR);
+
+ if(objectIdParts.length >= 2) {
+ ret = new
AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID],
objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]);
+ } else {
+ LOG.warn("Invalid soft-ref value: {}", softRefValue);
+ }
+ }
+
+ return ret;
+ }
+
+ public static List<AtlasObjectId> parseSoftRefValue(List<String>
softRefValue) {
+ List<AtlasObjectId> ret = null;
+
+ if (CollectionUtils.isNotEmpty(softRefValue)) {
+ ret = new ArrayList<>();
+
+ for (String elemValue : softRefValue) {
+ AtlasObjectId objId = parseSoftRefValue(elemValue);
+
+ if (objId != null) {
+ ret.add(objId);
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ public static Map<String, AtlasObjectId> parseSoftRefValue(Map<String,
String> softRefValue) {
+ Map<String, AtlasObjectId> ret = null;
+
+ if (MapUtils.isNotEmpty(softRefValue)) {
+ ret = new HashMap<>();
+
+ for (Map.Entry<String, String> entry : softRefValue.entrySet()) {
+ AtlasObjectId objId = parseSoftRefValue(entry.getValue());
+
+ if (objId != null) {
+ ret.put(entry.getKey(), objId);
+ }
+ }
+ }
+
+ return ret;
+ }
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index c57e30a..8170e1b 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -45,6 +45,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
@@ -210,14 +211,23 @@ public abstract class DeleteHandlerV1 {
TypeCategory typeCategory = attrType.getTypeCategory();
if (typeCategory == OBJECT_ID_TYPE) {
- AtlasEdge edge = graphHelper.getEdgeForLabel(vertex,
edgeLabel);
+ if (attributeInfo.getAttributeDef().isSoftReferenced()) {
+ String softRefVal =
vertex.getProperty(attributeInfo.getVertexPropertyName(), String.class);
+ AtlasObjectId refObjId =
AtlasEntityUtil.parseSoftRefValue(softRefVal);
+ AtlasVertex refVertex = refObjId != null ?
AtlasGraphUtilsV2.findByGuid(refObjId.getGuid()) : null;
- if (edge == null || getState(edge) == DELETED) {
- continue;
- }
+ if (refVertex != null) {
+ vertices.push(refVertex);
+ }
+ } else {
+ AtlasEdge edge = graphHelper.getEdgeForLabel(vertex,
edgeLabel);
- vertices.push(edge.getInVertex());
+ if (edge == null || getState(edge) == DELETED) {
+ continue;
+ }
+ vertices.push(edge.getInVertex());
+ }
} else if (typeCategory == ARRAY || typeCategory == MAP) {
TypeCategory elementType = null;
@@ -231,15 +241,46 @@ public abstract class DeleteHandlerV1 {
continue;
}
- List<AtlasEdge> edges =
getCollectionElementsUsingRelationship(vertex, attributeInfo);
+ if (attributeInfo.getAttributeDef().isSoftReferenced()) {
+ if (typeCategory == ARRAY) {
+ List softRefVal =
vertex.getListProperty(attributeInfo.getVertexPropertyName(), List.class);
+ List<AtlasObjectId> refObjIds =
AtlasEntityUtil.parseSoftRefValue(softRefVal);
+
+ if (CollectionUtils.isNotEmpty(refObjIds)) {
+ for (AtlasObjectId refObjId : refObjIds) {
+ AtlasVertex refVertex =
AtlasGraphUtilsV2.findByGuid(refObjId.getGuid());
+
+ if (refVertex != null) {
+ vertices.push(refVertex);
+ }
+ }
+ }
+ } else if (typeCategory == MAP) {
+ Map softRefVal =
vertex.getProperty(attributeInfo.getVertexPropertyName(), Map.class);
+ Map<String, AtlasObjectId> refObjIds =
AtlasEntityUtil.parseSoftRefValue(softRefVal);
+
+ if (MapUtils.isNotEmpty(refObjIds)) {
+ for (AtlasObjectId refObjId :
refObjIds.values()) {
+ AtlasVertex refVertex =
AtlasGraphUtilsV2.findByGuid(refObjId.getGuid());
- if (CollectionUtils.isNotEmpty(edges)) {
- for (AtlasEdge edge : edges) {
- if (edge == null || getState(edge) == DELETED) {
- continue;
+ if (refVertex != null) {
+ vertices.push(refVertex);
+ }
+ }
}
+ }
+
+ } else {
+ List<AtlasEdge> edges =
getCollectionElementsUsingRelationship(vertex, attributeInfo);
- vertices.push(edge.getInVertex());
+ if (CollectionUtils.isNotEmpty(edges)) {
+ for (AtlasEdge edge : edges) {
+ if (edge == null || getState(edge) == DELETED)
{
+ continue;
+ }
+
+ vertices.push(edge.getInVertex());
+ }
}
}
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index afa471f..967b052 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -54,6 +54,7 @@ import
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdg
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.utils.AtlasJson;
import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
import org.apache.commons.codec.digest.DigestUtils;
@@ -496,28 +497,27 @@ public class EntityGraphMapper {
}
}
- private Object mapSoftRefValue(AttributeMutationContext ctx,
EntityMutationContext context) {
- if(ctx.getValue() != null && !(ctx.getValue() instanceof
AtlasObjectId)) {
- LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found:
{}", ctx.getValue().getClass());
- return null;
- }
+ private String mapSoftRefValue(AttributeMutationContext ctx,
EntityMutationContext context) {
+ String ret = null;
- String softRefValue = null;
- if(ctx.getValue() != null) {
+ if (ctx.getValue() instanceof AtlasObjectId) {
AtlasObjectId objectId = (AtlasObjectId) ctx.getValue();
- String resolvedGuid =
AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid())
- ? context.getGuidAssignments().get(objectId.getGuid())
- : objectId.getGuid();
+ String typeName = objectId.getTypeName();
+ String guid =
AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid()) ?
context.getGuidAssignments().get(objectId.getGuid()) : objectId.getGuid();
- softRefValue = String.format(SOFT_REF_FORMAT,
objectId.getTypeName(), resolvedGuid);
+ ret = AtlasEntityUtil.formatSoftRefValue(typeName, guid);
+ } else {
+ if (ctx.getValue() != null) {
+ LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but
found: {}", ctx.getValue().getClass());
+ }
}
- return softRefValue;
+ return ret;
}
private Object mapSoftRefValueWithUpdate(AttributeMutationContext ctx,
EntityMutationContext context) {
+ String softRefValue = mapSoftRefValue(ctx, context);
- String softRefValue = (String) mapSoftRefValue(ctx, context);
AtlasGraphUtilsV2.setProperty(ctx.getReferringVertex(),
ctx.getVertexProperty(), softRefValue);
return softRefValue;
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index 912bc3e..a8024d6 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -52,6 +52,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.utils.AtlasJson;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
@@ -71,6 +72,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -106,9 +108,6 @@ public final class EntityGraphRetriever {
public static final String DESCRIPTION = "description";
public static final String OWNER = "owner";
public static final String CREATE_TIME = "createTime";
- private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":";
- private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
- private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1;
public static final String QUALIFIED_NAME = "qualifiedName";
private static final TypeReference<List<TimeBoundary>>
TIME_BOUNDARIES_LIST_TYPE = new TypeReference<List<TimeBoundary>>() {};
@@ -704,21 +703,21 @@ public final class EntityGraphRetriever {
break;
case OBJECT_ID_TYPE:
if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToObjectIdForSoftRef(entityVertex,
attribute.getVertexPropertyName());
+ ret = mapVertexToObjectIdForSoftRef(entityVertex,
attribute, entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToObjectId(entityVertex, edgeLabel,
null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
}
break;
case ARRAY:
if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToArrayForSoftRef(entityVertex,
attribute.getVertexPropertyName());
+ ret = mapVertexToArrayForSoftRef(entityVertex, attribute,
entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToArray(entityVertex, entityExtInfo,
isOwnedAttribute, attribute, isMinExtInfo);
}
break;
case MAP:
if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToMapForSoftRef(entityVertex,
attribute.getVertexPropertyName());
+ ret = mapVertexToMapForSoftRef(entityVertex, attribute,
entityExtInfo, isMinExtInfo);
} else {
ret = mapVertexToMap(entityVertex, entityExtInfo,
isOwnedAttribute, attribute, isMinExtInfo);
}
@@ -731,74 +730,78 @@ public final class EntityGraphRetriever {
return ret;
}
- private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex
entityVertex, String propertyName) {
- Map map = entityVertex.getProperty(propertyName, Map.class);
- if (MapUtils.isEmpty(map)) {
- return null;
- }
+ private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex
entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo,
final boolean isMinExtInfo) {
+ Map<String, AtlasObjectId> ret = null;
+ Map softRefVal =
entityVertex.getProperty(attribute.getVertexPropertyName(), Map.class);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Mapping map attribute {} for vertex {}", propertyName,
entityVertex);
- }
+ if (MapUtils.isNotEmpty(softRefVal)) {
+ ret = new HashMap<>();
- Map ret = new HashMap();
- for (Object mapKey : map.keySet()) {
- String softRefRaw = (String) map.get(mapKey);
- AtlasObjectId mapValue =
getAtlasObjectIdFromSoftRefFormat(softRefRaw);
- if (mapValue != null) {
- ret.put(mapKey, mapValue);
+ for (Object mapKey : softRefVal.keySet()) {
+ AtlasObjectId objectId =
getAtlasObjectIdFromSoftRefFormat(Objects.toString(softRefVal.get(mapKey)),
attribute, entityExtInfo, isMinExtInfo);
+
+ if (objectId != null) {
+ ret.put(Objects.toString(mapKey), objectId);
+ }
}
}
return ret;
}
- private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex
entityVertex, String propertyName) {
- List rawValue = entityVertex.getListProperty(propertyName, List.class);
- if (CollectionUtils.isEmpty(rawValue)) {
- return null;
- }
+ private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex
entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final
boolean isMinExtInfo) {
+ List<AtlasObjectId> ret = null;
+ List softRefVal =
entityVertex.getListProperty(attribute.getVertexPropertyName(), List.class);
- List list = (List) rawValue;
- List<AtlasObjectId> objectIds = new ArrayList<>();
- for (Object o : list) {
- if (!(o instanceof String)) {
- continue;
- }
+ if (CollectionUtils.isNotEmpty(softRefVal)) {
+ ret = new ArrayList<>();
- AtlasObjectId objectId =
getAtlasObjectIdFromSoftRefFormat((String) o);
- if(objectId == null) {
- continue;
- }
+ for (Object o : softRefVal) {
+ AtlasObjectId objectId =
getAtlasObjectIdFromSoftRefFormat(Objects.toString(o), attribute,
entityExtInfo, isMinExtInfo);
- objectIds.add(objectId);
+ if(objectId != null) {
+ ret.add(objectId);
+ }
+ }
}
- return objectIds;
+ return ret;
}
- private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex
entityVertex, String vertexPropertyName) {
- String rawValue = AtlasGraphUtilsV2.getEncodedProperty(entityVertex,
vertexPropertyName, String.class);
- if(StringUtils.isEmpty(rawValue)) {
- return null;
- }
+ private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex
entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final
boolean isMinExtInfo) {
+ String softRefVal = AtlasGraphUtilsV2.getEncodedProperty(entityVertex,
attribute.getVertexPropertyName(), String.class);
- return getAtlasObjectIdFromSoftRefFormat(rawValue);
+ return StringUtils.isNotEmpty(softRefVal) ?
getAtlasObjectIdFromSoftRefFormat(softRefVal, attribute, entityExtInfo,
isMinExtInfo) : null;
}
- private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String rawValue) {
- if(StringUtils.isEmpty(rawValue)) {
- return null;
- }
+ private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String softRefVal,
AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean
isMinExtInfo) {
+ AtlasObjectId ret = AtlasEntityUtil.parseSoftRefValue(softRefVal);
- String[] objectIdParts = StringUtils.split(rawValue,
SOFT_REFERENCE_FORMAT_SEPERATOR);
- if(objectIdParts.length < 2) {
- LOG.warn("Expecting value to be formatted for softRef. Instead
found: {}", rawValue);
- return null;
+ if(ret != null) {
+ if (entityExtInfo != null && attribute.isOwnedRef()) {
+ try {
+ AtlasVertex referenceVertex =
getEntityVertex(ret.getGuid());
+
+ if (referenceVertex != null) {
+ final AtlasEntity entity;
+
+ if (isMinExtInfo) {
+ entity =
mapVertexToAtlasEntityMin(referenceVertex, entityExtInfo);
+ } else {
+ entity = mapVertexToAtlasEntity(referenceVertex,
entityExtInfo);
+ }
+
+ if (entity != null) {
+ ret = toAtlasObjectId(entity);
+ }
+ }
+ } catch (AtlasBaseException excp) {
+ LOG.info("failed to retrieve soft-referenced
entity(typeName={}, guid={}); errorCode={}. Ignoring", ret.getTypeName(),
ret.getGuid(), excp.getAtlasErrorCode());
+ }
+ }
}
- return new
AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID],
- objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]);
+ return ret;
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex,
AtlasEntityExtInfo entityExtInfo,
@@ -986,7 +989,7 @@ public final class EntityGraphRetriever {
ret = AtlasTypeUtil.getAtlasObjectId(entity);
}
} else {
- ret = new AtlasObjectId(getGuid(referenceVertex),
getTypeName(referenceVertex));
+ ret = toAtlasObjectId(referenceVertex);
}
}
}
