http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java index df28ab3..e240fb6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java @@ -68,9 +68,9 @@ public final class GraphToTypedInstanceMapper { LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid); String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY); List<String> traits = GraphHelper.getTraitNames(instanceVertex); + String state = GraphHelper.getStateAsString(instanceVertex); - Id id = new Id(guid, instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName, - instanceVertex.<String>getProperty(Constants.STATE_PROPERTY_KEY)); + Id id = new Id(guid, instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName, state); LOG.debug("Created id {} for instance type {}", id, typeName); ClassType classType = typeSystem.getDataType(ClassType.class, typeName); @@ -161,9 +161,9 @@ public final class GraphToTypedInstanceMapper { Edge edge; if (edgeId == null) { - edge = GraphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);; + edge = GraphHelper.getEdgeForLabel(instanceVertex, relationshipLabel); } else { - edge = graphHelper.getEdgeById(edgeId); + edge = graphHelper.getEdgeByEdgeId(instanceVertex, relationshipLabel, edgeId); } if (edge != null) { @@ -175,9 +175,10 @@ public final class GraphToTypedInstanceMapper { LOG.debug("Found composite, mapping vertex to instance"); return mapGraphToTypedInstance(guid, referenceVertex); } else { + String state = GraphHelper.getStateAsString(referenceVertex); Id referenceId = new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), - dataType.getName()); + dataType.getName(), state); LOG.debug("Found non-composite, adding id {} ", referenceId); return referenceId; } @@ -271,7 +272,7 @@ public final class GraphToTypedInstanceMapper { if (edgeId == null) { edge = GraphHelper.getEdgeForLabel(instanceVertex, relationshipLabel); } else { - edge = graphHelper.getEdgeById(edgeId); + edge = graphHelper.getEdgeByEdgeId(instanceVertex, relationshipLabel, edgeId); } if (edge != null) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java index f8bbf73..3636791 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java @@ -26,20 +26,18 @@ import org.apache.atlas.typesystem.types.TypeSystem; public class HardDeleteHandler extends DeleteHandler { - private static final GraphHelper graphHelper = GraphHelper.getInstance(); - @Inject public HardDeleteHandler(TypeSystem typeSystem) { - super(typeSystem, true); + super(typeSystem, true, false); } @Override - protected void _deleteVertex(Vertex instanceVertex) { + protected void _deleteVertex(Vertex instanceVertex, boolean force) { graphHelper.removeVertex(instanceVertex); } @Override - protected void deleteEdge(Edge edge) throws AtlasException { + protected void deleteEdge(Edge edge, boolean force) throws AtlasException { graphHelper.removeEdge(edge); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java index aa78582..25aa7c5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java @@ -32,24 +32,34 @@ import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; public class SoftDeleteHandler extends DeleteHandler { @Inject public SoftDeleteHandler(TypeSystem typeSystem) { - super(typeSystem, false); + super(typeSystem, false, true); } @Override - protected void _deleteVertex(Vertex instanceVertex) { - Id.EntityState state = Id.EntityState.valueOf((String) instanceVertex.getProperty(STATE_PROPERTY_KEY)); - if (state != Id.EntityState.DELETED) { - GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name()); - GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); + protected void _deleteVertex(Vertex instanceVertex, boolean force) { + if (force) { + graphHelper.removeVertex(instanceVertex); + } else { + Id.EntityState state = GraphHelper.getState(instanceVertex); + if (state != Id.EntityState.DELETED) { + GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name()); + GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, + RequestContext.get().getRequestTime()); + } } } @Override - protected void deleteEdge(Edge edge) throws AtlasException { - Id.EntityState state = Id.EntityState.valueOf((String) edge.getProperty(STATE_PROPERTY_KEY)); - if (state != Id.EntityState.DELETED) { - GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name()); - GraphHelper.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); + protected void deleteEdge(Edge edge, boolean force) throws AtlasException { + if (force) { + graphHelper.removeEdge(edge); + } else { + Id.EntityState state = GraphHelper.getState(edge); + if (state != Id.EntityState.DELETED) { + GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name()); + GraphHelper + .setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime()); + } } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java index a017536..4c1f559 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java @@ -101,15 +101,15 @@ public final class TypedInstanceToGraphMapper { case CREATE: List<String> ids = addOrUpdateAttributesAndTraits(operation, entitiesToCreate); addFullTextProperty(entitiesToCreate, fulltextMapper); - requestContext.recordCreatedEntities(ids); + requestContext.recordEntityCreate(ids); break; case UPDATE_FULL: case UPDATE_PARTIAL: ids = addOrUpdateAttributesAndTraits(Operation.CREATE, entitiesToCreate); - requestContext.recordCreatedEntities(ids); + requestContext.recordEntityCreate(ids); ids = addOrUpdateAttributesAndTraits(operation, entitiesToUpdate); - requestContext.recordUpdatedEntities(ids); + requestContext.recordEntityUpdate(ids); addFullTextProperty(entitiesToCreate, fulltextMapper); addFullTextProperty(entitiesToUpdate, fulltextMapper); @@ -218,8 +218,8 @@ public final class TypedInstanceToGraphMapper { attrValue, currentEdge, edgeLabel, operation); if (currentEdge != null && !currentEdge.getId().toString().equals(newEdgeId)) { - deleteHandler.deleteReference(currentEdge, attributeInfo.dataType().getTypeCategory(), - attributeInfo.isComposite); + deleteHandler.deleteEdgeReference(currentEdge, attributeInfo.dataType().getTypeCategory(), + attributeInfo.isComposite, true); } break; @@ -326,7 +326,7 @@ public final class TypedInstanceToGraphMapper { String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo); List<String> currentElements = instanceVertex.getProperty(propertyName); IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType(); - List<String> newElementsCreated = new ArrayList<>(); + List<Object> newElementsCreated = new ArrayList<>(); if (!newAttributeEmpty) { if (newElements != null && !newElements.isEmpty()) { @@ -336,75 +336,53 @@ public final class TypedInstanceToGraphMapper { currentElements.get(index) : null; LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index, currentElement, newElements.get(index)); - String newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType, + Object newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType, newElements.get(index), currentElement, propertyName, operation); newElementsCreated.add(newEntry); } } } + List<String> additionalEdges = removeUnusedEntries(instanceVertex, propertyName, currentElements, + newElementsCreated, elementType, attributeInfo); + newElementsCreated.addAll(additionalEdges); + // for dereference on way out GraphHelper.setProperty(instanceVertex, propertyName, newElementsCreated); - - removeUnusedEntries(currentElements, newElementsCreated, elementType, attributeInfo); } - private void removeUnusedEntries(List<String> currentEntries, List<String> newEntries, IDataType entryType, - AttributeInfo attributeInfo) throws AtlasException { - if (currentEntries == null || currentEntries.isEmpty()) { - return; - } - - LOG.debug("Removing unused entries from the old collection"); - if (entryType.getTypeCategory() == DataTypes.TypeCategory.STRUCT - || entryType.getTypeCategory() == DataTypes.TypeCategory.CLASS) { - - //Get map of edge id to edge - Map<String, Edge> edgeMap = new HashMap<>(); - getEdges(currentEntries, edgeMap); - getEdges(newEntries, edgeMap); - - //Get final set of in vertices - Set<String> newInVertices = new HashSet<>(); - for (String edgeId : newEntries) { - Vertex inVertex = edgeMap.get(edgeId).getVertex(Direction.IN); - newInVertices.add(inVertex.getId().toString()); - } - - //Remove the edges for (current edges - new edges) - List<String> cloneElements = new ArrayList<>(currentEntries); - cloneElements.removeAll(newEntries); - LOG.debug("Removing unused entries from the old collection - {}", cloneElements); - - if (!cloneElements.isEmpty()) { - for (String edgeIdForDelete : cloneElements) { - Edge edge = edgeMap.get(edgeIdForDelete); - Vertex inVertex = edge.getVertex(Direction.IN); - if (newInVertices.contains(inVertex.getId().toString())) { - //If the edge.inVertex is in the new set of in vertices, just delete the edge - deleteHandler.deleteEdge(edge, true); - } else { - //else delete the edge + vertex - deleteHandler.deleteReference(edge, entryType.getTypeCategory(), attributeInfo.isComposite); + //Removes unused edges from the old collection, compared to the new collection + private List<String> removeUnusedEntries(Vertex instanceVertex, String edgeLabel, + Collection<String> currentEntries, + Collection<Object> newEntries, + IDataType entryType, AttributeInfo attributeInfo) throws AtlasException { + if (currentEntries != null && !currentEntries.isEmpty()) { + LOG.debug("Removing unused entries from the old collection"); + if (entryType.getTypeCategory() == DataTypes.TypeCategory.STRUCT + || entryType.getTypeCategory() == DataTypes.TypeCategory.CLASS) { + + //Remove the edges for (current edges - new edges) + List<String> cloneElements = new ArrayList<>(currentEntries); + cloneElements.removeAll(newEntries); + List<String> additionalElements = new ArrayList<>(); + LOG.debug("Removing unused entries from the old collection - {}", cloneElements); + + if (!cloneElements.isEmpty()) { + for (String edgeIdForDelete : cloneElements) { + Edge edge = graphHelper.getEdgeByEdgeId(instanceVertex, edgeLabel, edgeIdForDelete); + boolean deleted = deleteHandler.deleteEdgeReference(edge, entryType.getTypeCategory(), + attributeInfo.isComposite, true); + if (!deleted) { + additionalElements.add(edgeIdForDelete); + } } } + return additionalElements; } } + return new ArrayList<>(); } - private void getEdges(List<String> edgeIds, Map<String, Edge> edgeMap) { - if (edgeIds == null) { - return; - } - - for (String edgeId : edgeIds) { - if (!edgeMap.containsKey(edgeId)) { - edgeMap.put(edgeId, graphHelper.getEdgeById(edgeId)); - } - } - } - - /******************************************** MAP **************************************************/ private void mapMapCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex, @@ -421,38 +399,81 @@ public final class TypedInstanceToGraphMapper { IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType(); String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo); - List<String> currentElements = new ArrayList<>(); - List<String> newElementsCreated = new ArrayList<>(); - List<String> newKeysCreated = new ArrayList<>(); + + Map<String, String> currentMap = new HashMap<>(); + Map<String, Object> newMap = new HashMap<>(); + + List<String> currentKeys = instanceVertex.getProperty(propertyName); + if (currentKeys != null && !currentKeys.isEmpty()) { + for (String key : currentKeys) { + String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, key); + String propertyValueForKey = instanceVertex.getProperty(propertyNameForKey).toString(); + currentMap.put(key, propertyValueForKey); + } + } if (!newAttributeEmpty) { for (Map.Entry entry : newAttribute.entrySet()) { - String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, entry.getKey().toString()); - newKeysCreated.add(entry.getKey().toString()); + String keyStr = entry.getKey().toString(); + String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, keyStr); - String currentEntry = instanceVertex.getProperty(propertyNameForKey); - if (currentEntry != null) { - currentElements.add(currentEntry); - } - - String newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType, - entry.getValue(), currentEntry, propertyNameForKey, operation); + Object newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType, + entry.getValue(), currentMap.get(keyStr), propertyNameForKey, operation); //Add/Update/Remove property value GraphHelper.setProperty(instanceVertex, propertyNameForKey, newEntry); - newElementsCreated.add(newEntry); + newMap.put(keyStr, newEntry); } } + Map<String, String> additionalMap = + removeUnusedMapEntries(instanceVertex, propertyName, currentMap, newMap, elementType, attributeInfo); + + Set<String> newKeys = new HashSet<>(newMap.keySet()); + newKeys.addAll(additionalMap.keySet()); + // for dereference on way out - GraphHelper.setProperty(instanceVertex, propertyName, newKeysCreated); + GraphHelper.setProperty(instanceVertex, propertyName, new ArrayList<>(newKeys)); + } + + //Remove unused entries from map + private Map<String, String> removeUnusedMapEntries(Vertex instanceVertex, String propertyName, + Map<String, String> currentMap, + Map<String, Object> newMap, IDataType elementType, + AttributeInfo attributeInfo) + throws AtlasException { + boolean reference = (elementType.getTypeCategory() == DataTypes.TypeCategory.STRUCT + || elementType.getTypeCategory() == DataTypes.TypeCategory.CLASS); + Map<String, String> additionalMap = new HashMap<>(); + + for (String currentKey : currentMap.keySet()) { + boolean shouldDeleteKey = !newMap.containsKey(currentKey); + if (reference) { + String currentEdge = currentMap.get(currentKey); + //Delete the edge reference if its not part of new edges created/updated + if (!newMap.values().contains(currentEdge)) { + String edgeLabel = GraphHelper.getQualifiedNameForMapKey(propertyName, currentKey); + Edge edge = graphHelper.getEdgeByEdgeId(instanceVertex, edgeLabel, currentMap.get(currentKey)); + boolean deleted = + deleteHandler.deleteEdgeReference(edge, elementType.getTypeCategory(), attributeInfo.isComposite, true); + if (!deleted) { + additionalMap.put(currentKey, currentEdge); + shouldDeleteKey = false; + } + } + } - removeUnusedEntries(currentElements, newElementsCreated, elementType, attributeInfo); + if (shouldDeleteKey) { + String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, currentKey); + graphHelper.setProperty(instanceVertex, propertyNameForKey, null); + } + } + return additionalMap; } /******************************************** ARRAY & MAP **************************************************/ - private String addOrUpdateCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo, + private Object addOrUpdateCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo, IDataType elementType, Object newAttributeValue, String currentValue, String propertyName, Operation operation) throws AtlasException { @@ -460,7 +481,7 @@ public final class TypedInstanceToGraphMapper { switch (elementType.getTypeCategory()) { case PRIMITIVE: case ENUM: - return newAttributeValue != null ? newAttributeValue.toString() : null; + return newAttributeValue != null ? newAttributeValue : null; case ARRAY: case MAP: @@ -471,7 +492,7 @@ public final class TypedInstanceToGraphMapper { case STRUCT: case CLASS: final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName; - Edge currentEdge = graphHelper.getEdgeById(currentValue); + Edge currentEdge = graphHelper.getEdgeByEdgeId(instanceVertex, edgeLabel, currentValue); return addOrUpdateReference(instanceVertex, attributeInfo, elementType, newAttributeValue, currentEdge, edgeLabel, operation); @@ -526,7 +547,7 @@ public final class TypedInstanceToGraphMapper { mapInstanceToVertex(structInstance, structInstanceVertex, structInstance.fieldMapping().fields, false, Operation.CREATE); // add an edge to the newly created vertex from the parent - Edge newEdge = graphHelper.addEdge(instanceVertex, structInstanceVertex, edgeLabel); + Edge newEdge = graphHelper.getOrCreateEdge(instanceVertex, structInstanceVertex, edgeLabel); return newEdge; } @@ -575,7 +596,7 @@ public final class TypedInstanceToGraphMapper { private Edge addClassEdge(Vertex instanceVertex, Vertex toVertex, String edgeLabel) throws AtlasException { // add an edge to the class vertex from the instance - return graphHelper.addEdge(instanceVertex, toVertex, edgeLabel); + return graphHelper.getOrCreateEdge(instanceVertex, toVertex, edgeLabel); } private Vertex getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException { @@ -644,7 +665,7 @@ public final class TypedInstanceToGraphMapper { // add an edge to the newly created vertex from the parent String relationshipLabel = GraphHelper.getTraitLabel(entityType.getName(), traitName); - graphHelper.addEdge(parentInstanceVertex, traitInstanceVertex, relationshipLabel); + graphHelper.getOrCreateEdge(parentInstanceVertex, traitInstanceVertex, relationshipLabel); } /******************************************** PRIMITIVES **************************************************/ http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java index 3fb128c..5ed9e02 100755 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java @@ -69,6 +69,8 @@ public class GraphBackedTypeStore implements ITypeStore { private final TitanGraph titanGraph; + private GraphHelper graphHelper = GraphHelper.getInstance(); + @Inject public GraphBackedTypeStore(GraphProvider<TitanGraph> graphProvider) { titanGraph = graphProvider.get(); @@ -155,7 +157,7 @@ public class GraphBackedTypeStore implements ITypeStore { for (String superTypeName : superTypes) { HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName); Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription()); - addEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL); + graphHelper.getOrCreateEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL); } } } @@ -200,26 +202,11 @@ public class GraphBackedTypeStore implements ITypeStore { if (!coreTypes.contains(attrType.getName())) { Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription()); String label = getEdgeLabel(vertexTypeName, attribute.name); - addEdge(vertex, attrVertex, label); + graphHelper.getOrCreateEdge(vertex, attrVertex, label); } } } - private void addEdge(Vertex fromVertex, Vertex toVertex, String label) { - Iterator<Edge> edges = GraphHelper.getOutGoingEdgesByLabel(fromVertex, label); - // ATLAS-474: Check if this type system edge already exists, to avoid duplicates. - while (edges.hasNext()) { - Edge edge = edges.next(); - if (edge.getVertex(Direction.IN).equals(toVertex)) { - LOG.debug("Edge from {} to {} with label {} already exists", - toString(fromVertex), toString(toVertex), label); - return; - } - } - LOG.debug("Adding edge from {} to {} with label {}", toString(fromVertex), toString(toVertex), label); - titanGraph.addEdge(null, fromVertex, toVertex, label); - } - @Override @GraphTransaction public TypesDef restore() throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 7cd83f8..c1af0f6 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -26,6 +26,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.RequestContext; import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; @@ -58,8 +59,6 @@ import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.TypeUtils; -import org.apache.atlas.typesystem.types.TypeUtils.Pair; import org.apache.atlas.typesystem.types.ValueConversionException; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.utils.ParamChecker; @@ -305,16 +304,15 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang * Creates an entity, instance of the type. * * @param entityInstanceDefinition json array of entity definitions - * @return guids - json array of guids + * @return guids - list of guids */ @Override - public String createEntities(String entityInstanceDefinition) throws AtlasException { + public List<String> createEntities(String entityInstanceDefinition) throws AtlasException { ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition"); ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition); - List<String> guids = createEntities(typedInstances); - return new JSONArray(guids).toString(); + return createEntities(typedInstances); } public List<String> createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException { @@ -422,25 +420,26 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang * @return guids - json array of guids */ @Override - public String updateEntities(String entityInstanceDefinition) throws AtlasException { + public AtlasClient.EntityResult updateEntities(String entityInstanceDefinition) throws AtlasException { ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition"); ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition); - TypeUtils.Pair<List<String>, List<String>> guids = repository.updateEntities(typedInstances); - return onEntitiesAddedUpdated(guids); + AtlasClient.EntityResult entityResult = repository.updateEntities(typedInstances); + onEntitiesAddedUpdated(entityResult); + return entityResult; } - private String onEntitiesAddedUpdated(TypeUtils.Pair<List<String>, List<String>> guids) throws AtlasException { - onEntitiesAdded(guids.left); - onEntitiesUpdated(guids.right); - - guids.left.addAll(guids.right); - return new JSONArray(guids.left).toString(); + private void onEntitiesAddedUpdated(AtlasClient.EntityResult entityResult) throws AtlasException { + onEntitiesAdded(entityResult.getCreatedEntities()); + onEntitiesUpdated(entityResult.getUpdateEntities()); + //Note: doesn't access deletedEntities from entityResult + onEntitiesDeleted(RequestContext.get().getDeletedEntities()); } @Override - public String updateEntityAttributeByGuid(final String guid, String attributeName, String value) throws AtlasException { + public AtlasClient.EntityResult updateEntityAttributeByGuid(final String guid, String attributeName, + String value) throws AtlasException { ParamChecker.notEmpty(guid, "entity id"); ParamChecker.notEmpty(attributeName, "attribute name"); ParamChecker.notEmpty(value, "attribute value"); @@ -469,8 +468,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } ((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName())); - TypeUtils.Pair<List<String>, List<String>> guids = repository.updatePartial(newInstance); - return onEntitiesAddedUpdated(guids); + AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance); + onEntitiesAddedUpdated(entityResult); + return entityResult; } private ITypedReferenceableInstance validateEntityExists(String guid) @@ -483,7 +483,8 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } @Override - public String updateEntityPartialByGuid(final String guid, Referenceable newEntity) throws AtlasException { + public AtlasClient.EntityResult updateEntityPartialByGuid(final String guid, Referenceable newEntity) + throws AtlasException { ParamChecker.notEmpty(guid, "guid cannot be null"); ParamChecker.notNull(newEntity, "updatedEntity cannot be null"); ITypedReferenceableInstance existInstance = validateEntityExists(guid); @@ -491,11 +492,13 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang ITypedReferenceableInstance newInstance = convertToTypedInstance(newEntity, existInstance.getTypeName()); ((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName())); - TypeUtils.Pair<List<String>, List<String>> guids = repository.updatePartial(newInstance); - return onEntitiesAddedUpdated(guids); + AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance); + onEntitiesAddedUpdated(entityResult); + return entityResult; } - private ITypedReferenceableInstance convertToTypedInstance(Referenceable updatedEntity, String typeName) throws AtlasException { + private ITypedReferenceableInstance convertToTypedInstance(Referenceable updatedEntity, String typeName) + throws AtlasException { ClassType type = typeSystem.getDataType(ClassType.class, typeName); ITypedReferenceableInstance newInstance = type.createInstance(); @@ -538,8 +541,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } @Override - public String updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue, - Referenceable updatedEntity) throws AtlasException { + public AtlasClient.EntityResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, + String attrValue, + Referenceable updatedEntity) throws AtlasException { ParamChecker.notEmpty(typeName, "typeName"); ParamChecker.notEmpty(uniqueAttributeName, "uniqueAttributeName"); ParamChecker.notNull(attrValue, "unique attribute value"); @@ -550,8 +554,9 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang final ITypedReferenceableInstance newInstance = convertToTypedInstance(updatedEntity, typeName); ((ReferenceableInstance)newInstance).replaceWithNewId(oldInstance.getId()); - TypeUtils.Pair<List<String>, List<String>> guids = repository.updatePartial(newInstance); - return onEntitiesAddedUpdated(guids); + AtlasClient.EntityResult entityResult = repository.updatePartial(newInstance); + onEntitiesAddedUpdated(entityResult); + return entityResult; } private void validateTypeExists(String entityType) throws AtlasException { @@ -726,13 +731,14 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang * @see org.apache.atlas.services.MetadataService#deleteEntities(java.lang.String) */ @Override - public List<String> deleteEntities(List<String> deleteCandidateGuids) throws AtlasException { + public AtlasClient.EntityResult deleteEntities(List<String> deleteCandidateGuids) throws AtlasException { ParamChecker.notEmpty(deleteCandidateGuids, "delete candidate guids"); return deleteGuids(deleteCandidateGuids); } @Override - public List<String> deleteEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue) throws AtlasException { + public AtlasClient.EntityResult deleteEntityByUniqueAttribute(String typeName, String uniqueAttributeName, + String attrValue) throws AtlasException { ParamChecker.notEmpty(typeName, "delete candidate typeName"); ParamChecker.notEmpty(uniqueAttributeName, "delete candidate unique attribute name"); ParamChecker.notEmpty(attrValue, "delete candidate unique attribute value"); @@ -745,12 +751,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang return deleteGuids(deleteCandidateGuids); } - private List<String> deleteGuids(List<String> deleteCandidateGuids) throws AtlasException { - Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult = repository.deleteEntities(deleteCandidateGuids); - if (deleteEntitiesResult.right.size() > 0) { - onEntitiesDeleted(deleteEntitiesResult.right); - } - return deleteEntitiesResult.left; + private AtlasClient.EntityResult deleteGuids(List<String> deleteCandidateGuids) throws AtlasException { + AtlasClient.EntityResult entityResult = repository.deleteEntities(deleteCandidateGuids); + onEntitiesAddedUpdated(entityResult); + return entityResult; } private void onEntitiesDeleted(List<ITypedReferenceableInstance> entities) throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java index be407a5..f699404 100644 --- a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.audit; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.typesystem.Referenceable; import org.apache.commons.lang.RandomStringUtils; import org.testng.annotations.Test; @@ -38,7 +39,7 @@ public class AuditRepositoryTestBase { @Test public void testAddEvents() throws Exception { EntityAuditEvent event = new EntityAuditEvent(rand(), System.currentTimeMillis(), "u1", - EntityAuditEvent.EntityAuditAction.ENTITY_CREATE, "d1"); + EntityAuditEvent.EntityAuditAction.ENTITY_CREATE, "d1", new Referenceable(rand())); eventRepository.putEvents(event); @@ -54,17 +55,18 @@ public class AuditRepositoryTestBase { String id2 = "id2" + rand(); String id3 = "id3" + rand(); long ts = System.currentTimeMillis(); + Referenceable entity = new Referenceable(rand()); List<EntityAuditEvent> expectedEvents = new ArrayList<>(3); for (int i = 0; i < 3; i++) { //Add events for both ids EntityAuditEvent event = new EntityAuditEvent(id2, ts - i, "user" + i, - EntityAuditEvent.EntityAuditAction.ENTITY_UPDATE, "details" + i); + EntityAuditEvent.EntityAuditAction.ENTITY_UPDATE, "details" + i, entity); eventRepository.putEvents(event); expectedEvents.add(event); eventRepository.putEvents(new EntityAuditEvent(id1, ts - i, "user" + i, - EntityAuditEvent.EntityAuditAction.TAG_DELETE, "details" + i)); + EntityAuditEvent.EntityAuditAction.TAG_DELETE, "details" + i, entity)); eventRepository.putEvents(new EntityAuditEvent(id3, ts - i, "user" + i, - EntityAuditEvent.EntityAuditAction.TAG_ADD, "details" + i)); + EntityAuditEvent.EntityAuditAction.TAG_ADD, "details" + i, entity)); } //Use ts for which there is no event - ts + 2 http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java index 1aeedb5..449e066 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java @@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableSet; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Vertex; - import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.RepositoryMetadataModule; @@ -50,7 +49,6 @@ import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -60,7 +58,6 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -71,6 +68,7 @@ import java.util.Map; import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; import static org.apache.atlas.TestUtils.COLUMN_TYPE; import static org.apache.atlas.TestUtils.NAME; +import static org.apache.atlas.TestUtils.PII; import static org.apache.atlas.TestUtils.PROCESS_TYPE; import static org.apache.atlas.TestUtils.TABLE_TYPE; import static org.apache.atlas.TestUtils.createColumnEntity; @@ -78,8 +76,6 @@ import static org.apache.atlas.TestUtils.createDBEntity; import static org.apache.atlas.TestUtils.createTableEntity; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -145,7 +141,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { assertEquals(instance.getId()._getId(), id); //delete entity should mark it as deleted - List<String> results = deleteEntities(id); + List<String> results = deleteEntities(id).getDeletedEntities(); assertEquals(results.get(0), id); assertEntityDeleted(id); @@ -167,6 +163,26 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { } @Test + public void testDeleteEntityWithTraits() throws Exception { + Referenceable entity = createDBEntity(); + String id = createInstance(entity); + + TraitType dataType = typeSystem.getDataType(TraitType.class, PII); + ITypedStruct trait = dataType.convert(new Struct(TestUtils.PII), Multiplicity.REQUIRED); + repositoryService.addTrait(id, trait); + + ITypedReferenceableInstance instance = repositoryService.getEntityDefinition(id); + assertTrue(instance.getTraits().contains(PII)); + + deleteEntities(id); + assertEntityDeleted(id); + assertTestDeleteEntityWithTraits(id); + } + + protected abstract void assertTestDeleteEntityWithTraits(String guid) + throws EntityNotFoundException, RepositoryException, Exception; + + @Test public void testDeleteReference() throws Exception { //Deleting column should update table Referenceable db = createDBEntity(); @@ -179,13 +195,16 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { table.set(COLUMNS_ATTR_NAME, Arrays.asList(new Id(colId, 0, COLUMN_TYPE))); String tableId = createInstance(table); - deleteEntities(colId); + AtlasClient.EntityResult entityResult = deleteEntities(colId); + assertEquals(entityResult.getDeletedEntities().size(), 1); + assertEquals(entityResult.getDeletedEntities().get(0), colId); + assertEquals(entityResult.getUpdateEntities().size(), 1); + assertEquals(entityResult.getUpdateEntities().get(0), tableId); + assertEntityDeleted(colId); ITypedReferenceableInstance tableInstance = repositoryService.getEntityDefinition(tableId); - List<ITypedReferenceableInstance> columns = - (List<ITypedReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME); - assertNull(columns); + assertColumnForTestDeleteReference(tableInstance); //Deleting table should update process Referenceable process = new Referenceable(PROCESS_TYPE); @@ -195,18 +214,23 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { deleteEntities(tableId); assertEntityDeleted(tableId); - assertTestDeleteReference(processInstance); + + assertTableForTestDeleteReference(tableId); + assertProcessForTestDeleteReference(processInstance); } - protected abstract void assertTestDeleteReference(ITypedReferenceableInstance processInstance) throws Exception; + protected abstract void assertTableForTestDeleteReference(String tableId) throws Exception; + + protected abstract void assertColumnForTestDeleteReference(ITypedReferenceableInstance tableInstance) + throws AtlasException; + + protected abstract void assertProcessForTestDeleteReference(ITypedReferenceableInstance processInstance) throws Exception; protected abstract void assertEntityDeleted(String id) throws Exception; - private List<String> deleteEntities(String... id) throws Exception { + private AtlasClient.EntityResult deleteEntities(String... id) throws Exception { RequestContext.createContext(); - List<String> response = repositoryService.deleteEntities(Arrays.asList(id)).left; - assertNotNull(response); - return response; + return repositoryService.deleteEntities(Arrays.asList(id)); } private String createInstance(Referenceable entity) throws Exception { @@ -228,21 +252,41 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col3)); createInstance(table1Entity); - // Retrieve the table entities from the auditRepository, - // to get their guids and the composite column guids. + // Retrieve the table entities from the Repository, to get their guids and the composite column guids. ITypedReferenceableInstance tableInstance = repositoryService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, table1Entity.get(NAME)); - List<IReferenceableInstance> table1Columns = (List<IReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME); + List<IReferenceableInstance> columns = (List<IReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME); + + //Delete column + String colId = columns.get(0).getId()._getId(); + String tableId = tableInstance.getId()._getId(); + + AtlasClient.EntityResult entityResult = deleteEntities(colId); + assertEquals(entityResult.getDeletedEntities().size(), 1); + assertEquals(entityResult.getDeletedEntities().get(0), colId); + assertEquals(entityResult.getUpdateEntities().size(), 1); + assertEquals(entityResult.getUpdateEntities().get(0), tableId); + assertEntityDeleted(colId); + + tableInstance = repositoryService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, table1Entity.get(NAME)); + assertDeletedColumn(tableInstance); - // Delete the table entities. The deletion should cascade - // to their composite columns. - List<String> deletedGuids = deleteEntities(tableInstance.getId()._getId()); + //update by removing a column + tableInstance.set(COLUMNS_ATTR_NAME, ImmutableList.of(col3)); + entityResult = updatePartial(tableInstance); + colId = columns.get(1).getId()._getId(); + assertEquals(entityResult.getDeletedEntities().size(), 1); + assertEquals(entityResult.getDeletedEntities().get(0), colId); + assertEntityDeleted(colId); + + // Delete the table entities. The deletion should cascade to their composite columns. + tableInstance = repositoryService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, table1Entity.get(NAME)); + List<String> deletedGuids = deleteEntities(tableInstance.getId()._getId()).getDeletedEntities(); + assertEquals(deletedGuids.size(), 2); // Verify that deleteEntities() response has guids for tables and their composite columns. Assert.assertTrue(deletedGuids.contains(tableInstance.getId()._getId())); - for (IReferenceableInstance column : table1Columns) { - Assert.assertTrue(deletedGuids.contains(column.getId()._getId())); - } + Assert.assertTrue(deletedGuids.contains(columns.get(2).getId()._getId())); // Verify that tables and their composite columns have been deleted from the graph Repository. for (String guid : deletedGuids) { @@ -251,6 +295,8 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { assertTestDeleteEntities(tableInstance); } + protected abstract void assertDeletedColumn(ITypedReferenceableInstance tableInstance) throws AtlasException; + protected abstract void assertTestDeleteEntities(ITypedReferenceableInstance tableInstance) throws Exception; /** @@ -276,12 +322,13 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { vertexCount = getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance").size(); Assert.assertEquals(vertexCount, 1); - List<String> deletedEntities = deleteEntities(hrDeptGuid); + List<String> deletedEntities = deleteEntities(hrDeptGuid).getDeletedEntities(); assertTrue(deletedEntities.contains(hrDeptGuid)); + assertEntityDeleted(hrDeptGuid); // Verify Department entity and its contained Person entities were deleted. - assertEntityDeleted(hrDeptGuid); for (String employeeGuid : employeeGuids) { + assertTrue(deletedEntities.contains(employeeGuid)); assertEntityDeleted(employeeGuid); } @@ -341,15 +388,16 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey()); Assert.assertNotNull(object); - List<String> deletedEntities = deleteEntities(mapOwnerGuid); + List<String> deletedEntities = deleteEntities(mapOwnerGuid).getDeletedEntities(); Assert.assertEquals(deletedEntities.size(), 2); - Assert.assertTrue(deletedEntities.containsAll(guids)); + Assert.assertTrue(deletedEntities.contains(mapOwnerGuid)); + Assert.assertTrue(deletedEntities.contains(mapValueGuid)); assertEntityDeleted(mapOwnerGuid); assertEntityDeleted(mapValueGuid); } - private TypeUtils.Pair<List<String>, List<String>> updatePartial(ITypedReferenceableInstance entity) throws RepositoryException { + private AtlasClient.EntityResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException { RequestContext.createContext(); return repositoryService.updatePartial(entity); } @@ -379,7 +427,9 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { ClassType personType = typeSystem.getDataType(ClassType.class, "Person"); ITypedReferenceableInstance maxEntity = personType.createInstance(max.getId()); maxEntity.set("mentor", johnGuid); - updatePartial(maxEntity); + AtlasClient.EntityResult entityResult = updatePartial(maxEntity); + assertEquals(entityResult.getUpdateEntities().size(), 1); + assertTrue(entityResult.getUpdateEntities().contains(maxGuid)); // Verify the update was applied correctly - john should now be max's mentor. max = repositoryService.getEntityDefinition(maxGuid); @@ -394,7 +444,9 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { // Update max's mentor reference to jane. maxEntity.set("mentor", janeGuid); - updatePartial(maxEntity); + entityResult = updatePartial(maxEntity); + assertEquals(entityResult.getUpdateEntities().size(), 1); + assertTrue(entityResult.getUpdateEntities().contains(maxGuid)); // Verify the update was applied correctly - jane should now be max's mentor. max = repositoryService.getEntityDefinition(maxGuid); @@ -411,7 +463,11 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { Id juliusGuid = julius.getId(); maxEntity = personType.createInstance(max.getId()); maxEntity.set("manager", juliusGuid); - updatePartial(maxEntity); + entityResult = updatePartial(maxEntity); + //TODO ATLAS-499 should have updated julius' subordinates + assertEquals(entityResult.getUpdateEntities().size(), 2); + assertTrue(entityResult.getUpdateEntities().contains(maxGuid)); + assertTrue(entityResult.getUpdateEntities().contains(janeGuid._getId())); // Verify the update was applied correctly - julius should now be max's manager. max = repositoryService.getEntityDefinition(maxGuid); @@ -456,41 +512,38 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { } Assert.assertTrue(subordinateIds.contains(maxGuid)); - List<String> deletedEntities = deleteEntities(maxGuid); - Assert.assertTrue(deletedEntities.contains(maxGuid)); - assertEntityDeleted(maxGuid); - // Verify that the Department.employees reference to the deleted employee - // was disconnected. - hrDept = repositoryService.getEntityDefinition(hrDeptGuid); - refValue = hrDept.get("employees"); - Assert.assertTrue(refValue instanceof List); - List<Object> employees = (List<Object>)refValue; - Assert.assertEquals(employees.size(), 3); - for (Object listValue : employees) { - Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); - ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; - Assert.assertNotEquals(employee.getId()._getId(), maxGuid); - } + AtlasClient.EntityResult entityResult = deleteEntities(maxGuid); + ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Manager", "name", "John"); - // Verify that max's Person.mentor unidirectional reference to john was disconnected. - ITypedReferenceableInstance john = repositoryService.getEntityDefinition(johnGuid); - refValue = john.get("mentor"); - Assert.assertNull(refValue); + assertEquals(entityResult.getDeletedEntities().size(), 1); + assertTrue(entityResult.getDeletedEntities().contains(maxGuid)); + assertEquals(entityResult.getUpdateEntities().size(), 3); + assertTrue(entityResult.getUpdateEntities().containsAll(Arrays.asList(jane.getId()._getId(), hrDeptGuid, + john.getId()._getId()))); + assertEntityDeleted(maxGuid); - assertTestDisconnectBidirectionalReferences(janeGuid); + assertMaxForTestDisconnectBidirectionalReferences(nameGuidMap); // Now delete jane - this should disconnect the manager reference from her // subordinate. - deletedEntities = deleteEntities(janeGuid); - Assert.assertTrue(deletedEntities.contains(janeGuid)); + entityResult = deleteEntities(janeGuid); + assertEquals(entityResult.getDeletedEntities().size(), 1); + assertTrue(entityResult.getDeletedEntities().contains(janeGuid)); + assertEquals(entityResult.getUpdateEntities().size(), 2); + assertTrue(entityResult.getUpdateEntities().containsAll(Arrays.asList(hrDeptGuid, john.getId()._getId()))); + assertEntityDeleted(janeGuid); - john = repositoryService.getEntityDefinition(johnGuid); - Assert.assertNull(john.get("manager")); + john = repositoryService.getEntityDefinition("Person", "name", "John"); + assertJohnForTestDisconnectBidirectionalReferences(john, janeGuid); } - protected abstract void assertTestDisconnectBidirectionalReferences(String janeGuid) throws Exception; + protected abstract void assertJohnForTestDisconnectBidirectionalReferences(ITypedReferenceableInstance john, + String janeGuid) throws Exception; + + protected abstract void assertMaxForTestDisconnectBidirectionalReferences(Map<String, String> nameGuidMap) + throws Exception; /** * Verify deleting entity that is the target of a unidirectional class array reference @@ -503,30 +556,27 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { // Get the guid for one of the table's columns. ITypedReferenceableInstance table = repositoryService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", TestUtils.TABLE_NAME); String tableGuid = table.getId()._getId(); - Object refValues = table.get("columns"); - Assert.assertTrue(refValues instanceof List); - List<Object> refList = (List<Object>) refValues; - Assert.assertEquals(refList.size(), 5); - Assert.assertTrue(refList.get(0) instanceof ITypedReferenceableInstance); - ITypedReferenceableInstance column = (ITypedReferenceableInstance) refList.get(0); - String columnGuid = column.getId()._getId(); + List<ITypedReferenceableInstance> columns = (List<ITypedReferenceableInstance>) table.get("columns"); + Assert.assertEquals(columns.size(), 5); + String columnGuid = columns.get(0).getId()._getId(); // Delete the column. - List<String> deletedEntities = deleteEntities(columnGuid); - Assert.assertTrue(deletedEntities.contains(columnGuid)); + AtlasClient.EntityResult entityResult = deleteEntities(columnGuid); + assertEquals(entityResult.getDeletedEntities().size(), 1); + Assert.assertTrue(entityResult.getDeletedEntities().contains(columnGuid)); + assertEquals(entityResult.getUpdateEntities().size(), 1); + Assert.assertTrue(entityResult.getUpdateEntities().contains(tableGuid)); assertEntityDeleted(columnGuid); // Verify table.columns reference to the deleted column has been disconnected. table = repositoryService.getEntityDefinition(tableGuid); - refList = (List<Object>) table.get("columns"); - Assert.assertEquals(refList.size(), 4); - for (Object refValue : refList) { - Assert.assertTrue(refValue instanceof ITypedReferenceableInstance); - column = (ITypedReferenceableInstance)refValue; - Assert.assertFalse(column.getId()._getId().equals(columnGuid)); - } + assertTestDisconnectUnidirectionalArrayReferenceFromClassType( + (List<ITypedReferenceableInstance>) table.get("columns"), columnGuid); } + protected abstract void assertTestDisconnectUnidirectionalArrayReferenceFromClassType( + List<ITypedReferenceableInstance> columns, String columnGuid); + /** * Verify deleting entities that are the target of a unidirectional class array reference * from a struct or trait instance. @@ -559,8 +609,8 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { Referenceable structTargetEntity = new Referenceable("StructTarget"); Referenceable traitTargetEntity = new Referenceable("TraitTarget"); Referenceable structContainerEntity = new Referenceable("StructContainer"); - Referenceable structInstance = new Referenceable("TestStruct"); - Referenceable nestedStructInstance = new Referenceable("NestedStruct"); + Struct structInstance = new Struct("TestStruct"); + Struct nestedStructInstance = new Struct("NestedStruct"); Referenceable traitInstance = new Referenceable("TestTrait"); structContainerEntity.set("struct", structInstance); structInstance.set("target", ImmutableList.of(structTargetEntity)); @@ -623,19 +673,19 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { Assert.assertEquals(refList.get(0).getId()._getId(), traitTargetGuid); // Delete the entities that are targets of the struct and trait instances. - List<String> deletedEntities = deleteEntities(structTargetGuid, traitTargetGuid); + AtlasClient.EntityResult entityResult = deleteEntities(structTargetGuid, traitTargetGuid); + Assert.assertEquals(entityResult.getDeletedEntities().size(), 2); + Assert.assertTrue(entityResult.getDeletedEntities().containsAll(Arrays.asList(structTargetGuid, traitTargetGuid))); assertEntityDeleted(structTargetGuid); assertEntityDeleted(traitTargetGuid); - Assert.assertEquals(deletedEntities.size(), 2); - Assert.assertTrue(deletedEntities.containsAll(Arrays.asList(structTargetGuid, traitTargetGuid))); assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes(structContainerGuid); // Delete the entity which contains nested structs and has the TestTrait trait. - deletedEntities = deleteEntities(structContainerGuid); + entityResult = deleteEntities(structContainerGuid); + Assert.assertEquals(entityResult.getDeletedEntities().size(), 1); + Assert.assertTrue(entityResult.getDeletedEntities().contains(structContainerGuid)); assertEntityDeleted(structContainerGuid); - Assert.assertEquals(deletedEntities.size(), 1); - Assert.assertTrue(deletedEntities.contains(structContainerGuid)); // Verify all TestStruct struct vertices were removed. assertVerticesDeleted(getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "TestStruct")); @@ -890,13 +940,14 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { return list; } - private Map<String, String> getEmployeeNameGuidMap(ITypedReferenceableInstance hrDept) throws AtlasException { - + private Map<String, String> getEmployeeNameGuidMap(final ITypedReferenceableInstance hrDept) throws AtlasException { Object refValue = hrDept.get("employees"); Assert.assertTrue(refValue instanceof List); List<Object> employees = (List<Object>)refValue; Assert.assertEquals(employees.size(), 4); - Map<String, String> nameGuidMap = new HashMap<String, String>(); + Map<String, String> nameGuidMap = new HashMap<String, String>() {{ + put("hr", hrDept.getId()._getId()); + }}; for (Object listValue : employees) { Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java index d2109d3..cc60264 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java @@ -21,8 +21,10 @@ package org.apache.atlas.repository.graph; import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.AtlasClient; +import org.apache.atlas.AtlasException; import org.apache.atlas.TestUtils; import org.apache.atlas.repository.Constants; +import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; @@ -32,8 +34,11 @@ import org.apache.atlas.typesystem.types.TypeSystem; import org.testng.Assert; import java.util.List; +import java.util.Map; +import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.fail; import static org.testng.AssertJUnit.assertNotNull; @@ -45,7 +50,24 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo } @Override - protected void assertTestDeleteReference(ITypedReferenceableInstance processInstance) throws Exception { + protected void assertTestDeleteEntityWithTraits(String guid) { + //entity is deleted. So, no assertions + } + + @Override + protected void assertTableForTestDeleteReference(String tableId) { + //entity is deleted. So, no assertions + } + + @Override + protected void assertColumnForTestDeleteReference(ITypedReferenceableInstance tableInstance) throws AtlasException { + List<ITypedReferenceableInstance> columns = + (List<ITypedReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME); + assertNull(columns); + } + + @Override + protected void assertProcessForTestDeleteReference(ITypedReferenceableInstance processInstance) throws Exception { //assert that outputs is empty ITypedReferenceableInstance newProcess = repositoryService.getEntityDefinition(processInstance.getId()._getId()); @@ -63,6 +85,11 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo } @Override + protected void assertDeletedColumn(ITypedReferenceableInstance tableInstance) throws AtlasException { + assertEquals(((List<IReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME)).size(), 2); + } + + @Override protected void assertTestDeleteEntities(ITypedReferenceableInstance tableInstance) { int vertexCount = getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, TestUtils.TABLE_TYPE).size(); assertEquals(vertexCount, 0); @@ -85,12 +112,42 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo } @Override - protected void assertTestDisconnectBidirectionalReferences(String janeGuid) throws Exception { + protected void assertJohnForTestDisconnectBidirectionalReferences(ITypedReferenceableInstance john, + String janeGuid) throws Exception { + assertNull(john.get("manager")); + } + + @Override + protected void assertMaxForTestDisconnectBidirectionalReferences(Map<String, String> nameGuidMap) + throws Exception { + // Verify that the Department.employees reference to the deleted employee + // was disconnected. + ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(nameGuidMap.get("hr")); + List<ITypedReferenceableInstance> employees = (List<ITypedReferenceableInstance>) hrDept.get("employees"); + Assert.assertEquals(employees.size(), 3); + String maxGuid = nameGuidMap.get("Max"); + for (ITypedReferenceableInstance employee : employees) { + Assert.assertNotEquals(employee.getId()._getId(), maxGuid); + } + // Verify that the Manager.subordinates reference to the deleted employee // Max was disconnected. - ITypedReferenceableInstance jane = repositoryService.getEntityDefinition(janeGuid); + ITypedReferenceableInstance jane = repositoryService.getEntityDefinition(nameGuidMap.get("Jane")); List<ITypedReferenceableInstance> subordinates = (List<ITypedReferenceableInstance>) jane.get("subordinates"); assertEquals(subordinates.size(), 1); + + // Verify that max's Person.mentor unidirectional reference to john was disconnected. + ITypedReferenceableInstance john = repositoryService.getEntityDefinition(nameGuidMap.get("John")); + assertNull(john.get("mentor")); + } + + @Override + protected void assertTestDisconnectUnidirectionalArrayReferenceFromClassType( + List<ITypedReferenceableInstance> columns, String columnGuid) { + assertEquals(columns.size(), 4); + for (ITypedReferenceableInstance column : columns) { + assertFalse(column.getId()._getId().equals(columnGuid)); + } } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java index d9e3ec9..90bb635 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java @@ -19,10 +19,11 @@ package org.apache.atlas.repository.graph; import com.tinkerpop.blueprints.Vertex; - import org.apache.atlas.AtlasClient; +import org.apache.atlas.AtlasException; import org.apache.atlas.TestUtils; import org.apache.atlas.repository.Constants; +import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; @@ -33,8 +34,12 @@ import org.testng.Assert; import java.util.List; import java.util.Map; +import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; +import static org.apache.atlas.TestUtils.NAME; +import static org.apache.atlas.TestUtils.PII; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepositoryDeleteTestBase { @Override @@ -43,7 +48,38 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo } @Override - protected void assertTestDeleteReference(ITypedReferenceableInstance expected) throws Exception { + protected void assertTestDeleteEntityWithTraits(String guid) throws Exception { + ITypedReferenceableInstance instance = repositoryService.getEntityDefinition(guid); + assertTrue(instance.getTraits().contains(PII)); + } + + @Override + protected void assertTableForTestDeleteReference(String tableId) throws Exception { + ITypedReferenceableInstance table = repositoryService.getEntityDefinition(tableId); + assertNotNull(table.get(NAME)); + assertNotNull(table.get("description")); + assertNotNull(table.get("type")); + assertNotNull(table.get("tableType")); + assertNotNull(table.get("created")); + + Id dbId = (Id) table.get("database"); + assertNotNull(dbId); + + ITypedReferenceableInstance db = repositoryService.getEntityDefinition(dbId.getId()._getId()); + assertNotNull(db); + assertEquals(db.getId().getState(), Id.EntityState.ACTIVE); + } + + @Override + protected void assertColumnForTestDeleteReference(ITypedReferenceableInstance tableInstance) throws AtlasException { + List<ITypedReferenceableInstance> columns = + (List<ITypedReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME); + assertEquals(columns.size(), 1); + assertEquals(columns.get(0).getId().getState(), Id.EntityState.DELETED); + } + + @Override + protected void assertProcessForTestDeleteReference(ITypedReferenceableInstance expected) throws Exception { ITypedReferenceableInstance process = repositoryService.getEntityDefinition(expected.getId()._getId()); List<ITypedReferenceableInstance> outputs = (List<ITypedReferenceableInstance>) process.get(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS); @@ -59,6 +95,13 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo } @Override + protected void assertDeletedColumn(ITypedReferenceableInstance tableInstance) throws AtlasException { + List<IReferenceableInstance> columns = (List<IReferenceableInstance>) tableInstance.get(COLUMNS_ATTR_NAME); + assertEquals(columns.size(), 3); + assertEquals(columns.get(0).getId().getState(), Id.EntityState.DELETED); + } + + @Override protected void assertTestDeleteEntities(ITypedReferenceableInstance expected) throws Exception { //Assert that the deleted table can be fully constructed back ITypedReferenceableInstance table = repositoryService.getEntityDefinition(expected.getId()._getId()); @@ -67,6 +110,7 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo List<ITypedReferenceableInstance> expectedColumns = (List<ITypedReferenceableInstance>) table.get(TestUtils.COLUMNS_ATTR_NAME); assertEquals(columns.size(), expectedColumns.size()); + assertNotNull(table.get("database")); } @Override @@ -85,11 +129,57 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo } @Override - protected void assertTestDisconnectBidirectionalReferences(String janeGuid) throws Exception { + protected void assertJohnForTestDisconnectBidirectionalReferences(ITypedReferenceableInstance john, String janeGuid) + throws Exception { + Id mgr = (Id) john.get("manager"); + assertNotNull(mgr); + assertEquals(mgr._getId(), janeGuid); + assertEquals(mgr.getState(), Id.EntityState.DELETED); + } + + @Override + protected void assertMaxForTestDisconnectBidirectionalReferences(Map<String, String> nameGuidMap) throws Exception { + // Verify that the Department.employees reference to the deleted employee + // was disconnected. + ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(nameGuidMap.get("hr")); + List<ITypedReferenceableInstance> employees = (List<ITypedReferenceableInstance>) hrDept.get("employees"); + Assert.assertEquals(employees.size(), 4); + String maxGuid = nameGuidMap.get("Max"); + for (ITypedReferenceableInstance employee : employees) { + if (employee.getId()._getId().equals(maxGuid)) { + assertEquals(employee.getId().getState(), Id.EntityState.DELETED); + } + } + // Verify that the Manager.subordinates still references deleted employee - ITypedReferenceableInstance jane = repositoryService.getEntityDefinition(janeGuid); + ITypedReferenceableInstance jane = repositoryService.getEntityDefinition(nameGuidMap.get("Jane")); List<ITypedReferenceableInstance> subordinates = (List<ITypedReferenceableInstance>) jane.get("subordinates"); assertEquals(subordinates.size(), 2); + for (ITypedReferenceableInstance subordinate : subordinates) { + if (subordinate.getId()._getId().equals(maxGuid)) { + assertEquals(subordinate.getId().getState(), Id.EntityState.DELETED); + } + } + + // Verify that max's Person.mentor unidirectional reference to john was disconnected. + ITypedReferenceableInstance john = repositoryService.getEntityDefinition(nameGuidMap.get("John")); + Id mentor = (Id) john.get("mentor"); + assertEquals(mentor._getId(), maxGuid); + assertEquals(mentor.getState(), Id.EntityState.DELETED); + } + + @Override + protected void assertTestDisconnectUnidirectionalArrayReferenceFromClassType( + List<ITypedReferenceableInstance> columns, String columnGuid) { + Assert.assertEquals(columns.size(), 5); + for (ITypedReferenceableInstance column : columns) { + if (column.getId()._getId().equals(columnGuid)) { + assertEquals(column.getId().getState(), Id.EntityState.DELETED); + } else { + assertEquals(column.getId().getState(), Id.EntityState.ACTIVE); + } + } + } @Override @@ -122,7 +212,6 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo @Override protected void assertTestDeleteTargetOfMultiplicityRequiredReference() throws Exception { - // No-op - it's ok that no exception was thrown if soft deletes are enabled. } }
