Repository: incubator-atlas Updated Branches: refs/heads/master d2b9b99f4 -> 20608f022
ATLAS-372 Expose entity deletion through REST API (dkantor via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/20608f02 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/20608f02 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/20608f02 Branch: refs/heads/master Commit: 20608f0221108c2495820c3b098494dd66fe0854 Parents: d2b9b99 Author: Shwetha GS <[email protected]> Authored: Thu Feb 4 15:18:16 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Thu Feb 4 15:18:16 2016 +0530 ---------------------------------------------------------------------- .../main/java/org/apache/atlas/AtlasClient.java | 19 +++ .../notification/entity/EntityNotification.java | 1 + .../NotificationEntityChangeListener.java | 6 + release-log.txt | 1 + .../atlas/repository/MetadataRepository.java | 2 +- .../graph/GraphBackedMetadataRepository.java | 10 +- .../graph/TypedInstanceToGraphMapper.java | 32 ++++- .../atlas/services/DefaultMetadataService.java | 22 ++++ .../test/java/org/apache/atlas/TestUtils.java | 12 +- ...kedMetadataRepositoryDeleteEntitiesTest.java | 12 +- .../service/DefaultMetadataServiceTest.java | 130 +++++++++++++++++++ .../atlas/listener/EntityChangeListener.java | 8 ++ .../apache/atlas/services/MetadataService.java | 9 ++ .../atlas/web/resources/EntityResource.java | 39 +++++- .../web/resources/EntityJerseyResourceIT.java | 83 ++++++++++++ 15 files changed, 360 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/client/src/main/java/org/apache/atlas/AtlasClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java index 0a730af..1fc811a 100755 --- a/client/src/main/java/org/apache/atlas/AtlasClient.java +++ b/client/src/main/java/org/apache/atlas/AtlasClient.java @@ -166,6 +166,8 @@ public class AtlasClient { UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT, Response.Status.OK), UPDATE_ENTITY_PARTIAL(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK), LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), + DELETE_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK), + //Trait operations ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED), @@ -379,6 +381,23 @@ public class AtlasClient { } /** + * Delete the specified entities from the repository + * + * @param guids guids of entities to delete + * @return List of deleted entity guids + * @throws AtlasServiceException + */ + public List<String> deleteEntities(String ... guids) throws AtlasServiceException { + API api = API.DELETE_ENTITIES; + WebResource resource = getResource(api); + for (String guid : guids) { + resource = resource.queryParam(GUID.toLowerCase(), guid); + } + JSONObject jsonResponse = callAPIWithResource(API.DELETE_ENTITIES, resource); + return extractResults(jsonResponse, GUID); + } + + /** * Get an entity given the entity id * @param guid entity id * @return result object http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/notification/src/main/java/org/apache/atlas/notification/entity/EntityNotification.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/notification/entity/EntityNotification.java b/notification/src/main/java/org/apache/atlas/notification/entity/EntityNotification.java index 5579df2..82a1100 100644 --- a/notification/src/main/java/org/apache/atlas/notification/entity/EntityNotification.java +++ b/notification/src/main/java/org/apache/atlas/notification/entity/EntityNotification.java @@ -33,6 +33,7 @@ public interface EntityNotification { enum OperationType { ENTITY_CREATE, ENTITY_UPDATE, + ENTITY_DELETE, TRAIT_ADD, TRAIT_DELETE } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java b/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java index bdf9b2a..31f5c2b 100644 --- a/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java +++ b/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java @@ -76,9 +76,15 @@ public class NotificationEntityChangeListener implements EntityChangeListener { notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.TRAIT_DELETE); } + @Override + public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities) throws AtlasException { + notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_DELETE); + } + // ----- helper methods ------------------------------------------------- + // send notification of entity change private void notifyOfEntityEvent(Collection<ITypedReferenceableInstance> entityDefinitions, EntityNotification.OperationType operationType) throws AtlasException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 77d036b..b37c422 100644 --- a/release-log.txt +++ b/release-log.txt @@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-372 Expose entity deletion through REST API (dkantor via shwethags) ATLAS-452 Exceptions while running HiveHookIT#testAlterTableRename (shwethags) ATLAS-388 UI : On creating Tag, the page to be reset for creating new Tag (Anilg via shwethags) ATLAS-199 webapp build fails (grunt + tests) (sanjayp via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java index 4e86a0f..2fe42eb 100755 --- a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java @@ -110,7 +110,7 @@ public interface MetadataRepository { * @return guids of deleted entities * @throws RepositoryException */ - List <String> deleteEntities(String... guids) throws RepositoryException; + TypeUtils.Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntities(List<String> guids) throws RepositoryException; // Trait management functions http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index e002545..2c6e58b 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -30,16 +30,13 @@ import org.apache.atlas.GraphTransaction; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.repository.graph.TypedInstanceToGraphMapper.Operation; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.exception.EntityExistsException; import org.apache.atlas.typesystem.exception.EntityNotFoundException; -import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils; import org.slf4j.Logger; @@ -316,9 +313,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { @Override @GraphTransaction - public List<String> deleteEntities(String... guids) throws RepositoryException { + public TypeUtils.Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntities(List<String> guids) throws RepositoryException { - if (guids == null || guids.length == 0) { + if (guids == null || guids.size() == 0) { throw new IllegalArgumentException("guids must be non-null and non-empty"); } @@ -341,6 +338,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { throw new RepositoryException(e); } } - return instanceToGraphMapper.getDeletedEntities(); + return new TypeUtils.Pair<>( + instanceToGraphMapper.getDeletedEntityGuids(), instanceToGraphMapper.getDeletedEntities()); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java index 8668b46..7b2890c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java @@ -61,8 +61,8 @@ public final class TypedInstanceToGraphMapper { private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class); private final Map<Id, Vertex> idToVertexMap = new HashMap<>(); private final TypeSystem typeSystem = TypeSystem.getInstance(); - private final List<String> deletedEntities = new ArrayList<>(); - + private final List<String> deletedEntityGuids = new ArrayList<>(); + private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>(); private final GraphToTypedInstanceMapper graphToTypedInstanceMapper; private static final GraphHelper graphHelper = GraphHelper.getInstance(); @@ -688,7 +688,8 @@ public final class TypedInstanceToGraphMapper { // Remove any underlying structs and composite entities owned by this entity. mapInstanceToVertex(typedInstance, instanceVertex, classType.fieldMapping().fields, false, Operation.DELETE); - deletedEntities.add(id._getId()); + deletedEntityGuids.add(id._getId()); + deletedEntities.add(typedInstance); } /** @@ -728,12 +729,31 @@ public final class TypedInstanceToGraphMapper { /** - * Get the IDs of entities that have been deleted. + * Get the GUIDs of entities that have been deleted. * * @return */ - List<String> getDeletedEntities() { - return Collections.unmodifiableList(deletedEntities); + List<String> getDeletedEntityGuids() { + if (deletedEntityGuids.size() == 0) { + return Collections.emptyList(); + } + else { + return Collections.unmodifiableList(deletedEntityGuids); + } + } + + /** + * Get the entities that have been deleted. + * + * @return + */ + List<ITypedReferenceableInstance> getDeletedEntities() { + if (deletedEntities.size() == 0) { + return Collections.emptyList(); + } + else { + return Collections.unmodifiableList(deletedEntities); + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index b38face..2de8e50 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -21,6 +21,7 @@ package org.apache.atlas.services; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.inject.Provider; + import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.classification.InterfaceAudience; @@ -54,6 +55,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.TypeUtils.Pair; import org.apache.atlas.typesystem.types.ValueConversionException; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.utils.ParamChecker; @@ -65,6 +67,7 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Singleton; + import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -680,4 +683,23 @@ public class DefaultMetadataService implements MetadataService { public void unregisterListener(EntityChangeListener listener) { entityChangeListeners.remove(listener); } + + /* (non-Javadoc) + * @see org.apache.atlas.services.MetadataService#deleteEntities(java.lang.String) + */ + @Override + public List<String> deleteEntities(List<String> deleteCandidateGuids) throws AtlasException { + ParamChecker.notEmpty(deleteCandidateGuids, "delete candidate guids cannot be empty"); + Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult = repository.deleteEntities(deleteCandidateGuids); + if (deleteEntitiesResult.right.size() > 0) { + onEntitiesDeleted(deleteEntitiesResult.right); + } + return deleteEntitiesResult.left; + } + + private void onEntitiesDeleted(List<ITypedReferenceableInstance> entities) throws AtlasException { + for (EntityChangeListener listener : entityChangeListeners) { + listener.onEntitiesDeleted(entities); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index a77239c..653ec01 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -23,6 +23,7 @@ import com.thinkaurelius.titan.core.TitanGraph; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; + import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; @@ -187,6 +188,7 @@ public final class TestUtils { public static final String DATABASE_TYPE = "hive_database"; public static final String DATABASE_NAME = "foo"; public static final String TABLE_TYPE = "hive_table"; + public static final String COLUMN_TYPE = "column_type"; public static final String TABLE_NAME = "bar"; public static final String CLASSIFICATION = "classification"; public static final String PII = "PII"; @@ -220,7 +222,7 @@ public final class TestUtils { EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values); HierarchicalTypeDefinition<ClassType> columnsDefinition = - createClassTypeDef("column_type", ImmutableList.<String>of(), + createClassTypeDef(COLUMN_TYPE, ImmutableList.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE)); @@ -228,7 +230,7 @@ public final class TestUtils { new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),}); AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("cols", String.format("array<%s>", "column_type"), + new AttributeDefinition("cols", String.format("array<%s>", COLUMN_TYPE), Multiplicity.OPTIONAL, true, null), new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), @@ -256,7 +258,7 @@ public final class TestUtils { null), new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true, null), - new AttributeDefinition("columns", DataTypes.arrayTypeName("column_type"), + new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), Multiplicity.OPTIONAL, true, null), new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),}; @@ -277,7 +279,7 @@ public final class TestUtils { String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null), // array of classes - new AttributeDefinition("columns", String.format("array<%s>", "column_type"), + new AttributeDefinition("columns", String.format("array<%s>", COLUMN_TYPE), Multiplicity.OPTIONAL, true, null), // array of structs new AttributeDefinition("partitions", String.format("array<%s>", "partition_struct_type"), @@ -289,7 +291,7 @@ public final class TestUtils { //map of classes - new AttributeDefinition("columnsMap", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), - "column_type"), + COLUMN_TYPE), Multiplicity.OPTIONAL, true, null), //map of structs new AttributeDefinition("partitionsMap", http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java index e199913..b024152 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java @@ -21,6 +21,7 @@ package org.apache.atlas.repository.graph; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Vertex; + import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; @@ -32,6 +33,7 @@ import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.TypeUtils.Pair; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -39,7 +41,9 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; + import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -114,8 +118,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance"); Assert.assertEquals(vertexCount, 1); - List<String> deletedEntities = repositoryService.deleteEntities(hrDeptGuid); - Assert.assertTrue(deletedEntities.contains(hrDeptGuid)); + Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities = repositoryService.deleteEntities(Arrays.asList(hrDeptGuid)); + Assert.assertTrue(deletedEntities.left.contains(hrDeptGuid)); // Verify Department entity and its contained Person entities were deleted. verifyEntityDoesNotExist(hrDeptGuid); @@ -145,8 +149,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; String employeeGuid = employee.getId()._getId(); - List<String> deletedEntities = repositoryService.deleteEntities(employeeGuid); - Assert.assertTrue(deletedEntities.contains(employeeGuid)); + Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities = repositoryService.deleteEntities(Arrays.asList(employeeGuid)); + Assert.assertTrue(deletedEntities.left.contains(employeeGuid)); verifyEntityDoesNotExist(employeeGuid); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index 085888d..ea0d8e8 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -22,14 +22,20 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; + import org.apache.atlas.AtlasClient; import org.apache.atlas.typesystem.exception.TypeNotFoundException; import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.utils.ParamChecker; +import org.apache.atlas.AtlasException; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; +import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.atlas.typesystem.IStruct; +import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Struct; import org.apache.atlas.typesystem.TypesDef; @@ -49,6 +55,7 @@ import org.testng.annotations.Test; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -152,6 +159,12 @@ public class DefaultMetadataServiceTest { return entity; } + private Referenceable createColumnEntity() { + Referenceable entity = new Referenceable(TestUtils.COLUMN_TYPE); + entity.set("name", RandomStringUtils.randomAlphanumeric(10)); + entity.set("type", "VARCHAR(32)"); + return entity; + } @Test public void testCreateEntityWithUniqueAttribute() throws Exception { //name is the unique attribute @@ -685,4 +698,121 @@ public class DefaultMetadataServiceTest { //expected } } + + @Test + public void testDeleteEntities() throws Exception { + + + // Create 2 table entities, each with 3 composite column entities + Referenceable dbEntity = createDBEntity(); + String dbGuid = createInstance(dbEntity); + Id dbId = new Id(dbGuid, 0, TestUtils.DATABASE_TYPE); + Referenceable table1Entity = createTableEntity(dbId); + Referenceable table2Entity = createTableEntity(dbId); + Referenceable col1 = createColumnEntity(); + Referenceable col2 = createColumnEntity(); + Referenceable col3 = createColumnEntity(); + table1Entity.set("columns", ImmutableList.of(col1, col2, col3)); + table2Entity.set("columns", ImmutableList.of(col1, col2, col3)); + createInstance(table1Entity); + createInstance(table2Entity); + + // Retrieve the table entities from the repository, + // to get their guids and the composite column guids. + String entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, + "name", (String)table1Entity.get("name")); + Assert.assertNotNull(entityJson); + table1Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true); + Object val = table1Entity.get("columns"); + Assert.assertTrue(val instanceof List); + List<IReferenceableInstance> table1Columns = (List<IReferenceableInstance>) val; + entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, + "name", (String)table2Entity.get("name")); + Assert.assertNotNull(entityJson); + table2Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true); + val = table2Entity.get("columns"); + Assert.assertTrue(val instanceof List); + List<IReferenceableInstance> table2Columns = (List<IReferenceableInstance>) val; + + // Register an EntityChangeListener to verify the notification mechanism + // is working for deleteEntities(). + DeleteEntitiesChangeListener listener = new DeleteEntitiesChangeListener(); + metadataService.registerListener(listener); + + // Delete the table entities. The deletion should cascade + // to their composite columns. + JSONArray deleteCandidateGuids = new JSONArray(); + deleteCandidateGuids.put(table1Entity.getId()._getId()); + deleteCandidateGuids.put(table2Entity.getId()._getId()); + List<String> deletedGuids = metadataService.deleteEntities( + Arrays.asList(table1Entity.getId()._getId(), table2Entity.getId()._getId())); + + // Verify that deleteEntities() response has guids for tables and their composite columns. + Assert.assertTrue(deletedGuids.contains(table1Entity.getId()._getId())); + Assert.assertTrue(deletedGuids.contains(table2Entity.getId()._getId())); + for (IReferenceableInstance column : table1Columns) { + Assert.assertTrue(deletedGuids.contains(column.getId()._getId())); + } + for (IReferenceableInstance column : table2Columns) { + Assert.assertTrue(deletedGuids.contains(column.getId()._getId())); + } + + // Verify that tables and their composite columns have been deleted from the repository. + for (String guid : deletedGuids) { + try { + metadataService.getEntityDefinition(guid); + Assert.fail(EntityNotFoundException.class.getSimpleName() + + " expected but not thrown. The entity with guid " + guid + + " still exists in the repository after being deleted." ); + } + catch(EntityNotFoundException e) { + // The entity does not exist in the repository, so deletion was successful. + } + } + + // Verify that the listener was notified about the deleted entities. + Collection<ITypedReferenceableInstance> deletedEntitiesFromListener = listener.getDeletedEntities(); + Assert.assertNotNull(deletedEntitiesFromListener); + Assert.assertEquals(deletedEntitiesFromListener.size(), deletedGuids.size()); + List<String> deletedGuidsFromListener = new ArrayList<>(deletedGuids.size()); + for (ITypedReferenceableInstance deletedEntity : deletedEntitiesFromListener) { + deletedGuidsFromListener.add(deletedEntity.getId()._getId()); + } + Assert.assertEquals(deletedGuidsFromListener, deletedGuids); + } + + private static class DeleteEntitiesChangeListener implements EntityChangeListener { + + private Collection<ITypedReferenceableInstance> deletedEntities_; + + @Override + public void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) + throws AtlasException { + } + + @Override + public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) + throws AtlasException { + } + + @Override + public void onTraitAdded(ITypedReferenceableInstance entity, IStruct trait) + throws AtlasException { + } + + @Override + public void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) + throws AtlasException { + } + + @Override + public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities) + throws AtlasException { + deletedEntities_ = entities; + } + + public Collection<ITypedReferenceableInstance> getDeletedEntities() { + return deletedEntities_; + } + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java b/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java index 619ed85..4bf1d05 100644 --- a/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java +++ b/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java @@ -66,4 +66,12 @@ public interface EntityChangeListener { * @throws AtlasException if the listener notification fails */ void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) throws AtlasException; + + /** + * This is upon deleting entities from the repository. + * + * @param entities the deleted entities + * @throws AtlasException + */ + void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities) throws AtlasException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/server-api/src/main/java/org/apache/atlas/services/MetadataService.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java index a91c0a0..ab402d7 100644 --- a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java +++ b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java @@ -175,6 +175,15 @@ public interface MetadataService { void deleteTrait(String guid, String traitNameToBeDeleted) throws AtlasException; /** + * Delete the specified entities from the repository + * + * @param guids entity guids to be deleted + * @return List of guids for deleted entities + * @throws AtlasException + */ + List<String> deleteEntities(List<String> guids) throws AtlasException; + + /** * Register a listener for entity change. * * @param listener the listener to register http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java index f5ab4d8..e11b036 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java @@ -21,14 +21,14 @@ package org.apache.atlas.web.resources; import com.google.common.base.Preconditions; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; +import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.exception.EntityExistsException; import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.exception.TypeNotFoundException; -import org.apache.atlas.utils.ParamChecker; -import org.apache.atlas.services.MetadataService; -import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.types.ValueConversionException; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.web.util.Servlets; import org.apache.commons.lang.StringUtils; import org.codehaus.jettison.json.JSONArray; @@ -250,7 +250,7 @@ public class EntityResource { return updateEntityAttributeByGuid(guid, attribute, request); } } - + private Response updateEntityPartialByGuid(String guid, HttpServletRequest request) { try { ParamChecker.notEmpty(guid, "Guid property cannot be null"); @@ -312,6 +312,37 @@ public class EntityResource { } /** + * Delete entities from the repository + * + * @param guids deletion candidate guids + * @param request + * @return response payload as json + */ + @DELETE + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response deleteEntities(@QueryParam("guid") List<String> guids, @Context HttpServletRequest request) { + + try { + List<String> deletedGuids = metadataService.deleteEntities(guids); + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); + JSONArray guidArray = new JSONArray(deletedGuids.size()); + for (String guid : deletedGuids) { + guidArray.put(guid); + } + response.put(AtlasClient.GUID, guidArray); + return Response.ok(response).build(); + } catch (AtlasException | IllegalArgumentException e) { + LOG.error("Unable to delete entities {}", guids, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to delete entities {}", guids, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** * Fetch the complete definition of an entity given its GUID. * * @param guid GUID for the entity http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/20608f02/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java index f19463f..893099c 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; + import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.notification.NotificationConsumer; @@ -58,6 +59,7 @@ import org.testng.annotations.Test; import javax.ws.rs.HttpMethod; import javax.ws.rs.core.Response; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -698,4 +700,85 @@ public class EntityJerseyResourceIT extends BaseResourceIT { Assert.assertTrue(refs.get(0).equalsContents(columns.get(0))); Assert.assertTrue(refs.get(1).equalsContents(columns.get(1))); } + + @Test + public void testDeleteEntitiesViaRestApi() throws Exception { + // Create 2 database entities + Referenceable db1 = new Referenceable(DATABASE_TYPE); + db1.set("name", randomString()); + db1.set("description", randomString()); + Id db1Id = createInstance(db1); + Referenceable db2 = new Referenceable(DATABASE_TYPE); + db2.set("name", randomString()); + db2.set("description", randomString()); + Id db2Id = createInstance(db2); + + // Delete the database entities + ClientResponse clientResponse = service.path(ENTITIES). + queryParam(AtlasClient.GUID.toLowerCase(), db1Id._getId()). + queryParam(AtlasClient.GUID.toLowerCase(), db2Id._getId()). + accept(Servlets.JSON_MEDIA_TYPE).type(Servlets.JSON_MEDIA_TYPE).method(HttpMethod.DELETE, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + // Verify that response has guids for both database entities + JSONObject response = new JSONObject(clientResponse.getEntity(String.class)); + final String deletedGuidsJson = response.getString(AtlasClient.GUID); + Assert.assertNotNull(deletedGuidsJson); + JSONArray guidsArray = new JSONArray(deletedGuidsJson); + Assert.assertEquals(guidsArray.length(), 2); + List<String> deletedGuidsList = new ArrayList<>(2); + for (int index = 0; index < guidsArray.length(); index++) { + deletedGuidsList.add(guidsArray.getString(index)); + } + Assert.assertTrue(deletedGuidsList.contains(db1Id._getId())); + Assert.assertTrue(deletedGuidsList.contains(db2Id._getId())); + + // Verify entities were deleted from the repository. + for (String guid : deletedGuidsList) { + try { + serviceClient.getEntity(guid); + Assert.fail(AtlasServiceException.class.getSimpleName() + + " was expected but not thrown. The entity with guid " + guid + + " still exists in the repository after being deleted."); + } + catch (AtlasServiceException e) { + Assert.assertTrue(e.getMessage().contains(Integer.toString(Response.Status.NOT_FOUND.getStatusCode()))); + } + } + } + + @Test + public void testDeleteEntitiesViaClientApi() throws Exception { + // Create 2 database entities + Referenceable db1 = new Referenceable(DATABASE_TYPE); + db1.set("name", randomString()); + db1.set("description", randomString()); + Id db1Id = createInstance(db1); + Referenceable db2 = new Referenceable(DATABASE_TYPE); + db2.set("name", randomString()); + db2.set("description", randomString()); + Id db2Id = createInstance(db2); + + // Delete the database entities + List<String> deletedGuidsList = serviceClient.deleteEntities(db1Id._getId(), db2Id._getId()); + + // Verify that deleteEntities() response has database entity guids + Assert.assertEquals(deletedGuidsList.size(), 2); + Assert.assertTrue(deletedGuidsList.contains(db1Id._getId())); + Assert.assertTrue(deletedGuidsList.contains(db2Id._getId())); + + // Verify entities were deleted from the repository. + for (String guid : deletedGuidsList) { + try { + serviceClient.getEntity(guid); + Assert.fail(AtlasServiceException.class.getSimpleName() + + " was expected but not thrown. The entity with guid " + guid + + " still exists in the repository after being deleted."); + } + catch (AtlasServiceException e) { + Assert.assertTrue(e.getMessage().contains(Integer.toString(Response.Status.NOT_FOUND.getStatusCode()))); + } + } + } + }
