http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java index d6136ed..f18c308 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java @@ -257,7 +257,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { private String createInstance(Referenceable entity) throws Exception { ClassType dataType = typeSystem.getDataType(ClassType.class, entity.getTypeName()); ITypedReferenceableInstance instance = dataType.convert(entity, Multiplicity.REQUIRED); - List<String> results = repositoryService.createEntities(instance); + List<String> results = repositoryService.createEntities(instance).getCreatedEntities(); return results.get(results.size() - 1); } @@ -403,7 +403,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { ITypedReferenceableInstance mapValueInstance = compositeMapValueType.createInstance(); mapValueInstance.set(NAME, TestUtils.randomString()); mapOwnerInstance.set("map", Collections.singletonMap("value1", mapValueInstance)); - List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance); + List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance).getCreatedEntities(); Assert.assertEquals(createEntitiesResult.size(), 2); ITypedReferenceableInstance entityDefinition = repositoryService.getEntityDefinition("CompositeMapOwner", NAME, mapOwnerInstance.get(NAME)); @@ -412,7 +412,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { private AtlasClient.EntityResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException { RequestContext.createContext(); - return repositoryService.updatePartial(entity); + return repositoryService.updatePartial(entity).getEntityResult(); } @Test @@ -641,7 +641,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { structContainerType.convert(structContainerEntity, Multiplicity.REQUIRED); List<String> guids = repositoryService.createEntities( - structTargetConvertedEntity, traitTargetConvertedEntity, structContainerConvertedEntity); + structTargetConvertedEntity, traitTargetConvertedEntity, structContainerConvertedEntity).getCreatedEntities(); Assert.assertEquals(guids.size(), 3); guids = repositoryService.getEntityList("StructTarget"); @@ -746,7 +746,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { mapOwnerInstance.set("biMap", Collections.singletonMap("value1", mapValueInstance)); // Set biMapOwner reverse reference on MapValue. mapValueInstance.set("biMapOwner", mapOwnerInstance); - List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance); + List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance).getCreatedEntities(); Assert.assertEquals(createEntitiesResult.size(), 2); List<String> guids = repositoryService.getEntityList("MapOwner"); Assert.assertEquals(guids.size(), 1); @@ -856,7 +856,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { ITypedReferenceableInstance mapOwnerInstance = mapOwnerType.createInstance(); ITypedReferenceableInstance mapValueInstance = mapValueType.createInstance(); mapOwnerInstance.set("map", Collections.singletonMap("value1", mapValueInstance)); - List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance); + List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance).getCreatedEntities(); Assert.assertEquals(createEntitiesResult.size(), 2); List<String> guids = repositoryService.getEntityList("RequiredMapOwner"); Assert.assertEquals(guids.size(), 1); @@ -985,7 +985,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { // Create instance of MapValueReferencerContainer RequestContext.createContext(); ITypedReferenceableInstance mapValueReferencerContainer = mapValueReferencerContainerType.createInstance(); - List<String> createdEntities = repositoryService.createEntities(mapValueReferencerContainer); + List<String> createdEntities = repositoryService.createEntities(mapValueReferencerContainer).getCreatedEntities(); Assert.assertEquals(createdEntities.size(), 1); String mapValueReferencerContainerGuid = createdEntities.get(0); mapValueReferencerContainer = repositoryService.getEntityDefinition(createdEntities.get(0)); @@ -997,7 +997,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { mapValueReferencer.set("refToMapValue", mapValueInstance.getId()); RequestContext.createContext(); - EntityResult updateEntitiesResult = repositoryService.updateEntities(mapValueReferencerContainer); + EntityResult updateEntitiesResult = repositoryService.updateEntities(mapValueReferencerContainer).getEntityResult(); Assert.assertEquals(updateEntitiesResult.getCreatedEntities().size(), 1); Assert.assertEquals(updateEntitiesResult.getUpdateEntities().size(), 1); Assert.assertEquals(updateEntitiesResult.getUpdateEntities().get(0), mapValueReferencerContainerGuid); @@ -1016,7 +1016,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { public void testDeleteMixOfExistentAndNonExistentEntities() throws Exception { ITypedReferenceableInstance entity1 = compositeMapValueType.createInstance(); ITypedReferenceableInstance entity2 = compositeMapValueType.createInstance(); - List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2); + List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2).getCreatedEntities(); Assert.assertEquals(createEntitiesResult.size(), 2); List<String> guids = Arrays.asList(createEntitiesResult.get(0), "non-existent-guid1", "non-existent-guid2", createEntitiesResult.get(1)); EntityResult deleteEntitiesResult = repositoryService.deleteEntities(guids); @@ -1028,7 +1028,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { public void testDeleteMixOfNullAndNonNullGuids() throws Exception { ITypedReferenceableInstance entity1 = compositeMapValueType.createInstance(); ITypedReferenceableInstance entity2 = compositeMapValueType.createInstance(); - List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2); + List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2).getCreatedEntities(); Assert.assertEquals(createEntitiesResult.size(), 2); List<String> guids = Arrays.asList(createEntitiesResult.get(0), null, null, createEntitiesResult.get(1)); EntityResult deleteEntitiesResult = repositoryService.deleteEntities(guids); @@ -1076,7 +1076,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { ClassType dataType = typeSystem.getDataType(ClassType.class, table1Entity.getTypeName()); ITypedReferenceableInstance instance = dataType.convert(table1Entity, Multiplicity.REQUIRED); TestUtils.resetRequestContext(); - List<String> result = repositoryService.createEntities(instance); + List<String> result = repositoryService.createEntities(instance).getCreatedEntities(); Assert.assertEquals(result.size(), 3); ITypedReferenceableInstance entityDefinition = repositoryService.getEntityDefinition(TABLE_TYPE, NAME, tableName); String tableGuid = entityDefinition.getId()._getId(); @@ -1106,7 +1106,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { private String createHrDeptGraph() throws Exception { ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem); - List<String> guids = repositoryService.createEntities(hrDept); + List<String> guids = repositoryService.createEntities(hrDept).getCreatedEntities(); Assert.assertNotNull(guids); Assert.assertEquals(guids.size(), 5);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java index 96b0173..1ac9bd1 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -18,9 +18,29 @@ package org.apache.atlas.repository.graph; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import javax.inject.Inject; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.GraphTransaction; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RequestContext; @@ -64,28 +84,11 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Guice; import org.testng.annotations.Test; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import javax.inject.Inject; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.Arrays; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; -import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; - /** * GraphBackedMetadataRepository test * @@ -197,13 +200,190 @@ public class GraphBackedMetadataRepositoryTest { public void testSubmitEntity() throws Exception { ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem); - List<String> guids = repositoryService.createEntities(hrDept); + List<String> guids = repositoryService.createEntities(hrDept).getCreatedEntities(); Assert.assertNotNull(guids); Assert.assertEquals(guids.size(), 5); guid = guids.get(4); Assert.assertNotNull(guid); } + @Test + public void testCreateEntityWithOneNestingLevel() throws AtlasException { + + List<Referenceable> toValidate = new ArrayList<>(); + Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE); + toValidate.add(dept); + dept.set(TestUtils.NAME, "test1"); + + Referenceable mike = new Referenceable(TestUtils.PERSON_TYPE); + toValidate.add(mike); + + mike.set(TestUtils.NAME, "Mike"); + mike.set(TestUtils.DEPARTMENT_ATTR, dept); + + Referenceable mark = new Referenceable(TestUtils.PERSON_TYPE); + toValidate.add(mark); + mark.set(TestUtils.NAME, "Mark"); + mark.set(TestUtils.DEPARTMENT_ATTR, dept); + + dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(mike, mark)); + Map<String,Referenceable> positions = new HashMap<>(); + final String JANITOR = "janitor"; + final String RECEPTIONIST = "receptionist"; + positions.put(JANITOR, mike); + positions.put(RECEPTIONIST, mark); + dept.set(TestUtils.POSITIONS_ATTR, positions); + + + ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE); + ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED); + + CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance); + + System.out.println(result.getGuidMapping().toString()); + validateGuidMapping(toValidate, result); + } + + + @Test + public void testCreateEntityWithTwoNestingLevels() throws AtlasException { + + List<Referenceable> toVerify = new ArrayList<>(); + Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE); + toVerify.add(dept); + dept.set(TestUtils.NAME, "test2"); + + Referenceable wallace = new Referenceable(TestUtils.PERSON_TYPE); + toVerify.add(wallace); + wallace.set(TestUtils.NAME, "Wallace"); + wallace.set(TestUtils.DEPARTMENT_ATTR, dept); + + Referenceable wallaceComputer = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(wallaceComputer); + wallaceComputer.set("name", "wallaceComputer"); + wallace.set(TestUtils.ASSETS_ATTR, ImmutableList.of(wallaceComputer)); + + Referenceable jordan = new Referenceable(TestUtils.PERSON_TYPE); + toVerify.add(jordan); + jordan.set(TestUtils.NAME, "Jordan"); + jordan.set(TestUtils.DEPARTMENT_ATTR, dept); + + Referenceable jordanComputer = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(jordanComputer); + jordanComputer.set("name", "jordanComputer"); + jordan.set(TestUtils.ASSETS_ATTR, ImmutableList.of(jordanComputer)); + + dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(wallace, jordan)); + Map<String,Referenceable> positions = new HashMap<>(); + final String JANITOR = "janitor"; + final String RECEPTIONIST = "receptionist"; + positions.put(JANITOR, wallace); + positions.put(RECEPTIONIST, jordan); + dept.set(TestUtils.POSITIONS_ATTR, positions); + + + ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE); + ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED); + + CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance); + validateGuidMapping(toVerify, result); + } + + + @Test + public void testCreateEntityWithThreeNestingLevels() throws AtlasException { + + List<Referenceable> toVerify = new ArrayList<>(); + + Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE); + toVerify.add(dept); + dept.set(TestUtils.NAME, "test3"); + + Referenceable barry = new Referenceable(TestUtils.PERSON_TYPE); + toVerify.add(barry); + barry.set(TestUtils.NAME, "barry"); + barry.set(TestUtils.DEPARTMENT_ATTR, dept); + + Referenceable barryComputer = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(barryComputer); + barryComputer.set("name", "barryComputer"); + barry.set(TestUtils.ASSETS_ATTR, ImmutableList.of(barryComputer)); + + Referenceable barryHardDrive = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(barryHardDrive); + barryHardDrive.set("name", "barryHardDrive"); + + Referenceable barryCpuFan = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(barryCpuFan); + barryCpuFan.set("name", "barryCpuFan"); + + Referenceable barryVideoCard = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(barryVideoCard); + barryVideoCard.set("name", "barryVideoCard"); + + barryComputer.set("childAssets", ImmutableList.of(barryHardDrive, barryVideoCard, barryCpuFan)); + + + Referenceable jacob = new Referenceable(TestUtils.PERSON_TYPE); + toVerify.add(jacob); + jacob.set(TestUtils.NAME, "jacob"); + jacob.set(TestUtils.DEPARTMENT_ATTR, dept); + + Referenceable jacobComputer = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(jacobComputer); + jacobComputer.set("name", "jacobComputer"); + jacob.set(TestUtils.ASSETS_ATTR, ImmutableList.of(jacobComputer)); + + Referenceable jacobHardDrive = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(jacobHardDrive); + jacobHardDrive.set("name", "jacobHardDrive"); + + Referenceable jacobCpuFan = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(jacobCpuFan); + jacobCpuFan.set("name", "jacobCpuFan"); + + Referenceable jacobVideoCard = new Referenceable(TestUtils.ASSET_TYPE); + toVerify.add(jacobVideoCard); + jacobVideoCard.set("name", "jacobVideoCard"); + + jacobComputer.set("childAssets", ImmutableList.of(jacobHardDrive, jacobVideoCard, jacobCpuFan)); + + dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(barry, jacob)); + Map<String,Referenceable> positions = new HashMap<>(); + final String JANITOR = "janitor"; + final String RECEPTIONIST = "receptionist"; + positions.put(JANITOR, barry); + positions.put(RECEPTIONIST, jacob); + dept.set(TestUtils.POSITIONS_ATTR, positions); + + + ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE); + ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED); + + CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance); + + System.out.println(result.getGuidMapping().toString()); + assertEquals(result.getCreatedEntities().size(), toVerify.size()); + + validateGuidMapping(toVerify, result); + } + + private void validateGuidMapping(List<Referenceable> toVerify, CreateUpdateEntitiesResult result) + throws AtlasException { + Map<String,String> guids = result.getGuidMapping().getGuidAssignments(); + + TestUtils.assertContentsSame(result.getCreatedEntities(), guids.values()); + assertEquals(guids.size(), toVerify.size()); + for(Referenceable r : toVerify) { + loadAndDoSimpleValidation(guids.get(r.getId()._getId()), r); + } + } + + private ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable inst) throws AtlasException { + return TestUtils.loadAndDoSimpleValidation(guid, inst, repositoryService); + } + + @Test(dependsOnMethods = "testSubmitEntity") public void testGetEntityDefinitionForDepartment() throws Exception { ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid); @@ -279,7 +459,7 @@ public class GraphBackedMetadataRepositoryTest { private List<String> createEntities(ITypedReferenceableInstance... instances) throws Exception { RequestContext.createContext(); - return repositoryService.createEntities(instances); + return repositoryService.createEntities(instances).getCreatedEntities(); } private List<String> createEntity(Referenceable entity) throws Exception { @@ -385,7 +565,7 @@ public class GraphBackedMetadataRepositoryTest { Assert.assertEquals(traitNames.size(), 2); Assert.assertTrue(traitNames.contains(TestUtils.PII)); Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION)); - + // Verify modification timestamp was updated. GraphHelper.getInstance().getVertexForGUID(aGUID); Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); @@ -454,7 +634,7 @@ public class GraphBackedMetadataRepositoryTest { Assert.assertEquals(traitNames.size(), 2); Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION)); Assert.assertFalse(traitNames.contains(TestUtils.PII)); - + // Verify modification timestamp was updated. GraphHelper.getInstance().getVertexForGUID(aGUID); Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); @@ -729,7 +909,7 @@ public class GraphBackedMetadataRepositoryTest { ClassType deptType = typeSystem.getDataType(ClassType.class, "Department"); ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED); - List<String> guids = repositoryService.createEntities(hrDept2); + List<String> guids = repositoryService.createEntities(hrDept2).getCreatedEntities(); Assert.assertNotNull(guids); Assert.assertEquals(guids.size(), 2); Assert.assertNotNull(guids.get(0)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java index f5a6a05..3831920 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java @@ -143,7 +143,7 @@ public class GraphHelperTest { @Test public void testGetVerticesForGUIDSWithDuplicates() throws Exception { ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(TypeSystem.getInstance()); - List<String> result = repositoryService.createEntities(hrDept); + List<String> result = repositoryService.createEntities(hrDept).getCreatedEntities(); String guid = result.get(0); Map<String, AtlasVertex> verticesForGUIDs = GraphHelper.getInstance().getVerticesForGUIDs(Arrays.asList(guid, guid)); Assert.assertEquals(verticesForGUIDs.size(), 1); @@ -152,7 +152,7 @@ public class GraphHelperTest { @Test public void testGetCompositeGuidsAndVertices() throws Exception { ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem); - List<String> createdGuids = repositoryService.createEntities(hrDept); + List<String> createdGuids = repositoryService.createEntities(hrDept).getCreatedEntities(); String deptGuid = null; Set<String> expectedGuids = new HashSet<>(); @@ -214,7 +214,7 @@ public class GraphHelperTest { String entityjson = InstanceSerialization.toJson(entity, true); JSONArray entitiesJson = new JSONArray(); entitiesJson.put(entityjson); - List<String> guids = metadataService.createEntities(entitiesJson.toString()); + List<String> guids = metadataService.createEntities(entitiesJson.toString()).getCreatedEntities(); if (guids != null && guids.size() > 0) { return guids.get(guids.size() - 1); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java index a03f965..a2fffe7 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java @@ -19,10 +19,12 @@ package org.apache.atlas.repository.graph; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.GraphTransaction; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphQuery; import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; @@ -32,6 +34,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.exception.EntityExistsException; import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.IDataType; @@ -100,16 +103,21 @@ public class GraphRepoMapperScaleTest { ClassType dbType = typeSystem.getDataType(ClassType.class, TestUtils.DATABASE_TYPE); ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED); - dbGUID = repositoryService.createEntities(db).get(0); + dbGUID = result(db).getCreatedEntities().get(0); Referenceable dbInstance = new Referenceable(dbGUID, TestUtils.DATABASE_TYPE, databaseInstance.getValuesMap()); for (int index = 0; index < 1000; index++) { ITypedReferenceableInstance table = createHiveTableInstance(dbInstance, index); - repositoryService.createEntities(table); + result(table); } } + private CreateUpdateEntitiesResult result(ITypedReferenceableInstance db) + throws RepositoryException, EntityExistsException { + return repositoryService.createEntities(db); + } + @Test(dependsOnMethods = "testSubmitEntity") public void testSearchIndex() throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java index 265b316..c08bb88 100755 --- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java @@ -165,7 +165,8 @@ public class GraphBackedTypeStoreTest { HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description, ImmutableSet.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.OPTIONAL, - true, "department")); + true, "department"), + new AttributeDefinition("positions", String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL, false, null)); TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(deptTypeDef)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index 1ad79b1..6c791ba 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -170,7 +170,7 @@ public class DefaultMetadataServiceTest { String entityjson = InstanceSerialization.toJson(entity, true); JSONArray entitiesJson = new JSONArray(); entitiesJson.put(entityjson); - return metadataService.updateEntities(entitiesJson.toString()); + return metadataService.updateEntities(entitiesJson.toString()).getEntityResult(); } @Test(expectedExceptions = TypeNotFoundException.class) @@ -544,7 +544,7 @@ public class DefaultMetadataServiceTest { private AtlasClient.EntityResult updateEntityPartial(String guid, Referenceable entity) throws AtlasException { RequestContext.createContext(); - return metadataService.updateEntityPartialByGuid(guid, entity); + return metadataService.updateEntityPartialByGuid(guid, entity).getEntityResult(); } @Test http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/server-api/src/main/java/org/apache/atlas/services/MetadataService.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java index d5d8d9b..9b2bc9e 100644 --- a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java +++ b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java @@ -20,6 +20,7 @@ package org.apache.atlas.services; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.typesystem.ITypedReferenceableInstance; @@ -78,9 +79,9 @@ public interface MetadataService { * Creates an entity, instance of the type. * * @param entityDefinition definition - * @return json array of guids of entities created + * @return CreateUpdateEntitiesResult with the guids of the entities created */ - List<String> createEntities(String entityDefinition) throws AtlasException; + CreateUpdateEntitiesResult createEntities(String entityDefinition) throws AtlasException; /** * Get a typed entity instance. @@ -96,11 +97,11 @@ public interface MetadataService { * Create entity instances. * * @param typedInstances instance to create - * @return collection of guids for created entities + * @return CreateUpdateEntitiesResult with the guids of the entities created * * @throws AtlasException if unable to create the entities */ - List<String> createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException; + CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance[] typedInstances) throws AtlasException; /** @@ -148,36 +149,36 @@ public interface MetadataService { * @param guid entity id * @param attribute property name * @param value property value - * @return json array of guids of entities created/updated + * @return {@link CreateUpdateEntitiesResult} with the guids of the entities that were created/updated */ - AtlasClient.EntityResult updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException; + CreateUpdateEntitiesResult updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException; /** * Supports Partial updates of an entity. Users can update a subset of attributes for an entity identified by its guid * Note however that it cannot be used to set attribute values to null or delete attrbute values * @param guid entity id * @param entity - * @return json array of guids of entities created/updated + * @return {@link CreateUpdateEntitiesResult} with the guids of the entities that were created/updated * @throws AtlasException */ - AtlasClient.EntityResult updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException; + CreateUpdateEntitiesResult updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException; /** * Batch API - Adds/Updates the given entity id(guid). * * @param entityJson entity json - * @return json array of guids of entities created/updated + * @return {@link CreateUpdateEntitiesResult} with the guids of the entities that were created/updated */ - AtlasClient.EntityResult updateEntities(String entityJson) throws AtlasException; + CreateUpdateEntitiesResult updateEntities(String entityJson) throws AtlasException; /** * Batch API - Adds/Updates the given entity id(guid). * * @param entityJson entity json - * @return json array of guids of entities created/updated + * @return {@link CreateUpdateEntitiesResult} with the guids of the entities that were created/updated */ - AtlasClient.EntityResult updateEntities(ITypedReferenceableInstance[] iTypedReferenceableInstances) throws AtlasException; + CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance[] iTypedReferenceableInstances) throws AtlasException; // Trait management functions @@ -191,7 +192,7 @@ public interface MetadataService { * @return Guid of updated entity * @throws AtlasException */ - AtlasClient.EntityResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, + CreateUpdateEntitiesResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue, Referenceable updatedEntity) throws AtlasException; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java index 1dcad14..3d33b7f 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java @@ -618,7 +618,7 @@ public class TypeSystem { try { oldType = TypeSystem.this.getDataType(IDataType.class, newType.getName()); } catch (TypeNotFoundException e) { - LOG.debug("No existing type %s found - update OK", newType.getName()); + LOG.debug(String.format("No existing type %s found - update OK", newType.getName())); } if (oldType != null) { oldType.validateUpdate(newType); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java b/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java index 1559eb9..d8202a5 100755 --- a/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java +++ b/webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java @@ -356,7 +356,8 @@ public class QuickStartV2 { List<AtlasEntityHeader> entities = response.getEntitiesByOperation(EntityOperation.CREATE); if (CollectionUtils.isNotEmpty(entities)) { - ret = entitiesClient.getEntityByGuid(entities.get(0).getGuid()); + List<AtlasEntityWithAssociations> getByGuidResponse = entitiesClient.getEntityByGuid(entities.get(0).getGuid()); + ret = getByGuidResponse.get(0); System.out.println("Created entity of type [" + ret.getTypeName() + "], guid: " + ret.getGuid()); } @@ -567,7 +568,7 @@ public class QuickStartV2 { } private String getTableId(String tableName) throws AtlasServiceException { - AtlasEntity tableEntity = entitiesClient.getEntityByAttribute(TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName); + AtlasEntity tableEntity = entitiesClient.getEntityByAttribute(TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName).get(0); return tableEntity.getGuid(); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/main/java/org/apache/atlas/util/RestUtils.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/util/RestUtils.java b/webapp/src/main/java/org/apache/atlas/util/RestUtils.java index cecf636..9d4b04e 100644 --- a/webapp/src/main/java/org/apache/atlas/util/RestUtils.java +++ b/webapp/src/main/java/org/apache/atlas/util/RestUtils.java @@ -18,10 +18,22 @@ package org.apache.atlas.util; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; +import static org.apache.atlas.AtlasErrorCode.INVALID_TYPE_DEFINITION; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF; +import static org.apache.atlas.type.AtlasTypeUtil.isArrayType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; @@ -55,21 +67,12 @@ import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.apache.atlas.AtlasErrorCode.INVALID_TYPE_DEFINITION; -import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE; -import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE; -import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY; -import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF; -import static org.apache.atlas.type.AtlasTypeUtil.isArrayType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; public final class RestUtils { @@ -500,4 +503,5 @@ public final class RestUtils { private static boolean isEntity(AtlasType type) { return type.getTypeCategory() == TypeCategory.ENTITY; } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java b/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java index 7f5a056..6cdf9d1 100644 --- a/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java +++ b/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java @@ -17,11 +17,12 @@ */ package org.apache.atlas.web.adapters; -import com.google.inject.Inject; -import com.google.inject.Singleton; +import java.util.List; + import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasClassification; @@ -30,6 +31,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasEntityWithAssociations; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.model.instance.GuidMapping; import org.apache.atlas.services.MetadataService; import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; @@ -50,7 +52,8 @@ import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; -import java.util.List; +import com.google.inject.Inject; +import com.google.inject.Singleton; import java.util.Map; @Singleton @@ -142,8 +145,14 @@ public class AtlasInstanceRestAdapters { return ctx.getEntities(); } + public static EntityMutationResponse toEntityMutationResponse(AtlasClient.EntityResult entityResult) { - public static EntityMutationResponse toEntityMutationResponse(AtlasClient.EntityResult result) { + CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); + result.setEntityResult(entityResult); + return toEntityMutationResponse(result); + } + + public static EntityMutationResponse toEntityMutationResponse(CreateUpdateEntitiesResult result) { EntityMutationResponse response = new EntityMutationResponse(); for (String guid : result.getCreatedEntities()) { AtlasEntityHeader header = new AtlasEntityHeader(); @@ -151,7 +160,7 @@ public class AtlasInstanceRestAdapters { response.addEntity(EntityMutations.EntityOperation.CREATE, header); } - for (String guid : result.getUpdateEntities()) { + for (String guid : result.getUpdatedEntities()) { AtlasEntityHeader header = new AtlasEntityHeader(); header.setGuid(guid); response.addEntity(EntityMutations.EntityOperation.UPDATE, header); @@ -162,6 +171,10 @@ public class AtlasInstanceRestAdapters { header.setGuid(guid); response.addEntity(EntityMutations.EntityOperation.DELETE, header); } + GuidMapping guidMapping = result.getGuidMapping(); + if(guidMapping != null) { + response.setGuidAssignments(guidMapping.getGuidAssignments()); + } return response; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java index 9ca684d..b90627f 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java @@ -23,7 +23,10 @@ import com.google.common.base.Preconditions; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.AtlasClient.EntityResult; +import org.apache.atlas.model.instance.GuidMapping; import org.apache.atlas.services.MetadataService; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.Referenceable; @@ -127,13 +130,13 @@ public class EntityResource { LOG.debug("submitting entities {} ", entityJson); } - final List<String> guids = metadataService.createEntities(entities); - + final CreateUpdateEntitiesResult result = metadataService.createEntities(entities); + final List<String> guids = result.getEntityResult().getCreatedEntities(); if (LOG.isDebugEnabled()) { LOG.debug("Created entities {}", guids); } - JSONObject response = getResponse(new AtlasClient.EntityResult(guids, null, null)); + JSONObject response = getResponse(result); URI locationURI = getLocationURI(guids); @@ -179,13 +182,26 @@ public class EntityResource { } private JSONObject getResponse(AtlasClient.EntityResult entityResult) throws AtlasException, JSONException { + CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); + result.setEntityResult(entityResult); + return getResponse(result); + + } + private JSONObject getResponse(CreateUpdateEntitiesResult result) throws AtlasException, JSONException { JSONObject response = new JSONObject(); + EntityResult entityResult = result.getEntityResult(); + GuidMapping mapping = result.getGuidMapping(); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.ENTITIES, new JSONObject(entityResult.toString()).get(AtlasClient.ENTITIES)); - String sampleEntityId = getSample(entityResult); - if (sampleEntityId != null) { - String entityDefinition = metadataService.getEntityDefinitionJson(sampleEntityId); - response.put(AtlasClient.DEFINITION, new JSONObject(entityDefinition)); + if(entityResult != null) { + response.put(AtlasClient.ENTITIES, new JSONObject(entityResult.toString()).get(AtlasClient.ENTITIES)); + String sampleEntityId = getSample(result.getEntityResult()); + if (sampleEntityId != null) { + String entityDefinition = metadataService.getEntityDefinitionJson(sampleEntityId); + response.put(AtlasClient.DEFINITION, new JSONObject(entityDefinition)); + } + } + if(mapping != null) { + response.put(AtlasClient.GUID_ASSIGNMENTS, new JSONObject(mapping.toString()).get(AtlasClient.GUID_ASSIGNMENTS)); } return response; } @@ -218,13 +234,13 @@ public class EntityResource { LOG.info("updating entities {} ", entityJson); } - AtlasClient.EntityResult entityResult = metadataService.updateEntities(entities); + CreateUpdateEntitiesResult result = metadataService.updateEntities(entities); if (LOG.isDebugEnabled()) { - LOG.debug("Updated entities: {}", entityResult); + LOG.debug("Updated entities: {}", result.getEntityResult()); } - JSONObject response = getResponse(entityResult); + JSONObject response = getResponse(result); return Response.ok(response).build(); } catch(EntityExistsException e) { LOG.error("Unique constraint violation for entityDef={}", entityJson, e); @@ -303,14 +319,14 @@ public class EntityResource { Referenceable updatedEntity = InstanceSerialization.fromJsonReferenceable(entityJson, true); - AtlasClient.EntityResult entityResult = + CreateUpdateEntitiesResult result = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, updatedEntity); if (LOG.isDebugEnabled()) { - LOG.debug("Updated entities: {}", entityResult); + LOG.debug("Updated entities: {}", result.getEntityResult()); } - JSONObject response = getResponse(entityResult); + JSONObject response = getResponse(result); return Response.ok(response).build(); } catch (ValueConversionException ve) { LOG.error("Unable to persist entity instance due to a deserialization error {} ", entityJson, ve); @@ -388,13 +404,13 @@ public class EntityResource { Referenceable updatedEntity = InstanceSerialization.fromJsonReferenceable(entityJson, true); - AtlasClient.EntityResult entityResult = metadataService.updateEntityPartialByGuid(guid, updatedEntity); + CreateUpdateEntitiesResult result = metadataService.updateEntityPartialByGuid(guid, updatedEntity); if (LOG.isDebugEnabled()) { - LOG.debug("Updated entities: {}", entityResult); + LOG.debug("Updated entities: {}", result.getEntityResult()); } - JSONObject response = getResponse(entityResult); + JSONObject response = getResponse(result); return Response.ok(response).build(); } catch (EntityNotFoundException e) { LOG.error("An entity with GUID={} does not exist {} ", guid, entityJson, e); @@ -429,13 +445,13 @@ public class EntityResource { LOG.debug("Updating entity {} for property {} = {}", guid, property, value); } - AtlasClient.EntityResult entityResult = metadataService.updateEntityAttributeByGuid(guid, property, value); + CreateUpdateEntitiesResult result = metadataService.updateEntityAttributeByGuid(guid, property, value); if (LOG.isDebugEnabled()) { - LOG.debug("Updated entities: {}", entityResult); + LOG.debug("Updated entities: {}", result.getEntityResult()); } - JSONObject response = getResponse(entityResult); + JSONObject response = getResponse(result); return Response.ok(response).build(); } catch (EntityNotFoundException e) { LOG.error("An entity with GUID={} does not exist {} ", guid, value, e); @@ -453,7 +469,7 @@ public class EntityResource { * Delete entities from the repository identified by their guids (including their composite references) * or * Deletes a single entity identified by its type and unique attribute value from the repository (including their composite references) - * + * * @param guids list of deletion candidate guids * or * @param entityType the entity type http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/main/java/org/apache/atlas/web/rest/EntitiesREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntitiesREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntitiesREST.java index af0377e..24c2151 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntitiesREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntitiesREST.java @@ -20,6 +20,7 @@ package org.apache.atlas.web.rest; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.SearchFilter; import org.apache.atlas.model.instance.AtlasClassification; @@ -100,7 +101,7 @@ public class EntitiesREST { ITypedReferenceableInstance[] entitiesInOldFormat = restAdapters.getITypedReferenceables(entities.values()); try { - final AtlasClient.EntityResult result = metadataService.updateEntities(entitiesInOldFormat); + final CreateUpdateEntitiesResult result = metadataService.updateEntities(entitiesInOldFormat); response = toEntityMutationResponse(result); } catch (AtlasException e) { LOG.error("Exception while getting a typed reference for the entity ", e); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 9c0ccf6..bca9091 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -20,6 +20,7 @@ package org.apache.atlas.web.rest; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; +import org.apache.atlas.CreateUpdateEntitiesResult; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasClassification; @@ -151,7 +152,7 @@ public class EntityREST { AtlasFormatConverter.ConverterContext ctx = new AtlasFormatConverter.ConverterContext(); ctx.addEntity(entity); Referenceable ref = restAdapters.getReferenceable(entity, ctx); - AtlasClient.EntityResult result = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, ref); + CreateUpdateEntitiesResult result = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, ref); return toEntityMutationResponse(result); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java b/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java index 9528139..b7cc273 100644 --- a/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java +++ b/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java @@ -56,7 +56,7 @@ public class QuickStartV2IT extends BaseResourceIT { } private AtlasEntity getDB(String dbName) throws AtlasServiceException, JSONException { - AtlasEntity dbEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.DATABASE_TYPE, "name", dbName); + AtlasEntity dbEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.DATABASE_TYPE, "name", dbName).get(0); return dbEntity; } @@ -73,14 +73,15 @@ public class QuickStartV2IT extends BaseResourceIT { } private AtlasEntity getTable(String tableName) throws AtlasServiceException { - AtlasEntity tableEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName); + AtlasEntity tableEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName).get(0); return tableEntity; } private AtlasEntity getProcess(String processName) throws AtlasServiceException { - AtlasEntity processEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.LOAD_PROCESS_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processName); + AtlasEntity processEntity = entitiesClientV2.getEntityByAttribute(QuickStartV2.LOAD_PROCESS_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processName).get(0); return processEntity; } + private void verifyTrait(AtlasEntity table) throws AtlasServiceException { AtlasClassification.AtlasClassifications classfications = entitiesClientV2.getClassifications(table.getGuid()); @@ -115,7 +116,7 @@ public class QuickStartV2IT extends BaseResourceIT { @Test public void testProcessIsAdded() throws AtlasServiceException, JSONException { AtlasEntity loadProcess = entitiesClientV2.getEntityByAttribute(QuickStartV2.LOAD_PROCESS_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, - QuickStartV2.LOAD_SALES_DAILY_PROCESS); + QuickStartV2.LOAD_SALES_DAILY_PROCESS).get(0); Map loadProcessAttribs = loadProcess.getAttributes(); assertEquals(QuickStartV2.LOAD_SALES_DAILY_PROCESS, loadProcessAttribs.get(AtlasClient.NAME)); @@ -168,7 +169,7 @@ public class QuickStartV2IT extends BaseResourceIT { @Test public void testViewIsAdded() throws AtlasServiceException, JSONException { - AtlasEntity view = entitiesClientV2.getEntityByAttribute(QuickStartV2.VIEW_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, QuickStartV2.PRODUCT_DIM_VIEW); + AtlasEntity view = entitiesClientV2.getEntityByAttribute(QuickStartV2.VIEW_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, QuickStartV2.PRODUCT_DIM_VIEW).get(0); Map<String, Object> viewAttributes = view.getAttributes(); assertEquals(QuickStartV2.PRODUCT_DIM_VIEW, viewAttributes.get(AtlasClient.NAME)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java b/webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java new file mode 100644 index 0000000..0cb112a --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/util/RestUtilsTest.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.util; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; +import org.apache.atlas.repository.store.graph.v1.AtlasStructDefStoreV1; +import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +/** + * Validates that conversion from V1 to legacy types (and back) is consistent. This also tests + * that the conversion logic in AtlasStructDefStoreV1 is consistent with the conversion logic + * in RestUtils. This tests particularly focuses on composite attributes, since a defect was + * found in that area. + */ +public class RestUtilsTest { + + @Test(enabled=false) + // FIXME: On conversion back to V1, reverse attribute name + // "containingDatabase" + // in tables attribute in "database" type is lost. See ATLAS-1528. + public void testBidirectonalCompositeMappingConsistent() throws AtlasBaseException { + + HierarchicalTypeDefinition<ClassType> dbV1Type = TypesUtil.createClassTypeDef("database", + ImmutableSet.<String> of(), new AttributeDefinition("tables", DataTypes.arrayTypeName("table"), + Multiplicity.OPTIONAL, true, "containingDatabase")); + + HierarchicalTypeDefinition<ClassType> tableV1Type = TypesUtil.createClassTypeDef("table", + ImmutableSet.<String> of(), + new AttributeDefinition("containingDatabase", "database", Multiplicity.OPTIONAL, false, "tables")); + + testV1toV2toV1Conversion(Arrays.asList(dbV1Type, tableV1Type), new boolean[] { true, false }); + } + + @Test(enabled=false) + // FIXME: On conversion back to V1, reverse attribute name + // "containingDatabase" is lost + // in "table" attribute in "database". See ATLAS-1528. + public void testBidirectonalNonCompositeMappingConsistent() throws AtlasBaseException { + + HierarchicalTypeDefinition<ClassType> dbV1Type = TypesUtil.createClassTypeDef("database", + ImmutableSet.<String> of(), new AttributeDefinition("tables", DataTypes.arrayTypeName("table"), + Multiplicity.OPTIONAL, false, "containingDatabase")); + + HierarchicalTypeDefinition<ClassType> tableV1Type = TypesUtil.createClassTypeDef("table", + ImmutableSet.<String> of(), + new AttributeDefinition("containingDatabase", "database", Multiplicity.OPTIONAL, false, "tables")); + + testV1toV2toV1Conversion(Arrays.asList(dbV1Type, tableV1Type), new boolean[] { false, false }); + } + + private AtlasTypeDefGraphStoreV1 makeTypeStore(AtlasTypeRegistry reg) { + + AtlasTypeDefGraphStoreV1 result = mock(AtlasTypeDefGraphStoreV1.class); + + for (AtlasEntityType type : reg.getAllEntityTypes()) { + String typeName = type.getTypeName(); + AtlasVertex typeVertex = mock(AtlasVertex.class); + when(result.isTypeVertex(eq(typeVertex), any(TypeCategory.class))).thenReturn(true); + when(typeVertex.getProperty(eq(Constants.TYPE_CATEGORY_PROPERTY_KEY), eq(TypeCategory.class))) + .thenReturn(TypeCategory.CLASS); + + String attributeListPropertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(typeName); + when(typeVertex.getProperty(eq(attributeListPropertyKey), eq(List.class))) + .thenReturn(new ArrayList<>(type.getAllAttributes().keySet())); + for (AtlasAttribute attribute : type.getAllAttributes().values()) { + String attributeDefPropertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(typeName, attribute.getName()); + String attributeJson = AtlasStructDefStoreV1.toJsonFromAttribute(attribute); + when(typeVertex.getProperty(eq(attributeDefPropertyKey), eq(String.class))).thenReturn(attributeJson); + } + when(result.findTypeVertexByName(eq(typeName))).thenReturn(typeVertex); + } + return result; + } + + private AtlasAttributeDef convertToJsonAndBack(AtlasTypeRegistry registry, AtlasStructDef structDef, + AtlasAttributeDef attributeDef, boolean compositeExpected) throws AtlasBaseException { + + AtlasTypeDefGraphStoreV1 typeDefStore = makeTypeStore(registry); + AtlasStructType structType = (AtlasStructType) registry.getType(structDef.getName()); + AtlasAttribute attribute = structType.getAttribute(attributeDef.getName()); + String attribJson = AtlasStructDefStoreV1.toJsonFromAttribute(attribute); + + Map attrInfo = AtlasType.fromJson(attribJson, Map.class); + Assert.assertEquals(attrInfo.get("isComposite"), compositeExpected); + return AtlasStructDefStoreV1.toAttributeDefFromJson(structDef, attrInfo, typeDefStore); + } + + private void testV1toV2toV1Conversion(List<HierarchicalTypeDefinition<ClassType>> typesToTest, + boolean[] compositeExpected) throws AtlasBaseException { + + List<AtlasEntityDef> convertedEntityDefs = convertV1toV2(typesToTest); + + AtlasTypeRegistry registry = createRegistry(convertedEntityDefs); + for(int i = 0 ; i < convertedEntityDefs.size(); i++) { + AtlasEntityDef def = convertedEntityDefs.get(i); + for (AtlasAttributeDef attrDef : def.getAttributeDefs()) { + AtlasAttributeDef converted = convertToJsonAndBack(registry, def, attrDef, compositeExpected[i]); + Assert.assertEquals(converted, attrDef); + } + } + + List<HierarchicalTypeDefinition<ClassType>> convertedBackTypeDefs = convertV2toV1(convertedEntityDefs); + + for (int i = 0; i < typesToTest.size(); i++) { + + HierarchicalTypeDefinition<ClassType> convertedBack = convertedBackTypeDefs.get(i); + Assert.assertEquals(convertedBack, typesToTest.get(i)); + AttributeDefinition[] attributeDefinitions = convertedBack.attributeDefinitions; + if (attributeDefinitions.length > 0) { + Assert.assertEquals(attributeDefinitions[0].isComposite, compositeExpected[i]); + } + } + + } + + private List<HierarchicalTypeDefinition<ClassType>> convertV2toV1(List<AtlasEntityDef> toConvert) + throws AtlasBaseException { + + AtlasTypeRegistry reg = createRegistry(toConvert); + + List<HierarchicalTypeDefinition<ClassType>> result = new ArrayList<>(toConvert.size()); + for (int i = 0; i < toConvert.size(); i++) { + AtlasEntityDef entityDef = toConvert.get(i); + AtlasEntityType entity = reg.getEntityTypeByName(entityDef.getName()); + HierarchicalTypeDefinition<ClassType> converted = RestUtils.toTypesDef(entity, reg) + .classTypesAsJavaList().get(0); + result.add(converted); + } + return result; + } + + private AtlasTypeRegistry createRegistry(List<AtlasEntityDef> toConvert) throws AtlasBaseException { + AtlasTypeRegistry reg = new AtlasTypeRegistry(); + AtlasTransientTypeRegistry tmp = reg.lockTypeRegistryForUpdate(); + tmp.addTypes(toConvert); + reg.releaseTypeRegistryForUpdate(tmp, true); + return reg; + } + + private List<AtlasEntityDef> convertV1toV2(List<HierarchicalTypeDefinition<ClassType>> types) + throws AtlasBaseException { + + ImmutableList<HierarchicalTypeDefinition<ClassType>> classTypeList = ImmutableList + .<HierarchicalTypeDefinition<ClassType>> builder().addAll(types).build(); + + TypesDef toConvert = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition> of(), + ImmutableList.<StructTypeDefinition> of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>> of(), + classTypeList); + + String json = TypesSerialization.toJson(toConvert); + AtlasTypeRegistry emptyRegistry = new AtlasTypeRegistry(); + AtlasTypesDef converted = RestUtils.toAtlasTypesDef(json, emptyRegistry); + List<AtlasEntityDef> convertedEntityDefs = converted.getEntityDefs(); + return convertedEntityDefs; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java index 18bbc3b..6187cfb 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java @@ -18,10 +18,18 @@ package org.apache.atlas.web.resources; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import kafka.consumer.ConsumerTimeoutException; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasDiscoveryClientV2; @@ -40,6 +48,9 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.notification.NotificationConsumer; import org.apache.atlas.notification.entity.EntityNotification; @@ -50,7 +61,17 @@ import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.*; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.EnumValue; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.utils.AuthenticationUtil; import org.apache.atlas.utils.ParamChecker; @@ -62,14 +83,11 @@ import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.BeforeClass; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; +import kafka.consumer.ConsumerTimeoutException; /** * Base class for integration tests. @@ -125,6 +143,45 @@ public abstract class BaseResourceIT { } } + protected void batchCreateTypes(AtlasTypesDef typesDef) throws AtlasServiceException { + for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) { + try { + typedefClientV2.createEnumDef(enumDef); + } catch (AtlasServiceException ex) { + LOG.warn("EnumDef creation failed for {}", enumDef.getName()); + } + } + for (AtlasStructDef structDef : typesDef.getStructDefs()) { + try { + typedefClientV2.createStructDef(structDef); + } catch (AtlasServiceException ex) { + LOG.warn("StructDef creation failed for {}", structDef.getName()); + } + } + + AtlasTypesDef entityDefs = new AtlasTypesDef( + Collections.<AtlasEnumDef>emptyList(), + Collections.<AtlasStructDef>emptyList(), + Collections.<AtlasClassificationDef>emptyList(), + typesDef.getEntityDefs()); + try { + typedefClientV2.createAtlasTypeDefs(entityDefs); + } + catch(AtlasServiceException e) { + LOG.warn("Type creation failed for {}", typesDef.toString()); + LOG.warn(e.toString()); + } + + for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) { + try { + typedefClientV2.createClassificationDef(classificationDef); + } catch (AtlasServiceException ex) { + LOG.warn("ClassificationDef creation failed for {}", classificationDef.getName()); + } + } + + } + protected void createType(AtlasTypesDef typesDef) { // Since the bulk create bails out on a single failure, this has to be done as a workaround for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) { @@ -182,7 +239,7 @@ public abstract class BaseResourceIT { } LOG.info("Types already exist. Skipping type creation"); } catch(AtlasServiceException ase) { - //Expected if type doesnt exist + //Expected if type doesn't exist String typesAsJSON = TypesSerialization.toJson(typesDef); createType(typesAsJSON); } @@ -278,7 +335,10 @@ public abstract class BaseResourceIT { TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), TypesUtil.createRequiredAttrDef(DESCRIPTION, DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE), - attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.INT_TYPE)); + attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.INT_TYPE), + new AttributeDefinition("tables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE), + Multiplicity.OPTIONAL, false, "db") + ); HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil .createClassTypeDef(COLUMN_TYPE, null, attrDef(NAME, DataTypes.STRING_TYPE), @@ -297,7 +357,7 @@ public abstract class BaseResourceIT { attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE), attrDef("lastAccessTime", DataTypes.DATE_TYPE), attrDef("temporary", DataTypes.BOOLEAN_TYPE), - new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, true, null), + new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.OPTIONAL, true, "tables"), new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), Multiplicity.OPTIONAL, true, null), new AttributeDefinition("tableType", "tableType", Multiplicity.OPTIONAL, false, null), @@ -347,6 +407,13 @@ public abstract class BaseResourceIT { } protected void createTypeDefinitionsV2() throws Exception { + + AtlasConstraintDef isCompositeSourceConstraint = new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, + Collections.<String, Object>singletonMap(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE)); + + AtlasConstraintDef isCompositeTargetConstraint = new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, + Collections.<String, Object>emptyMap()); + AtlasEntityDef dbClsTypeDef = AtlasTypeUtil.createClassTypeDef( DATABASE_TYPE_V2, null, @@ -354,7 +421,15 @@ public abstract class BaseResourceIT { AtlasTypeUtil.createRequiredAttrDef(DESCRIPTION, "string"), AtlasTypeUtil.createOptionalAttrDef("locationUri", "string"), AtlasTypeUtil.createOptionalAttrDef("owner", "string"), - AtlasTypeUtil.createOptionalAttrDef("createTime", "int")); + AtlasTypeUtil.createOptionalAttrDef("createTime", "int"), + AtlasTypeUtil.createOptionalAttrDef("createTime", "int"), + //there is a serializ + new AtlasAttributeDef("randomTable", + DataTypes.arrayTypeName(HIVE_TABLE_TYPE_V2), + true, + Cardinality.SET, + 0, -1, false, true, Collections.singletonList(isCompositeSourceConstraint)) + ); AtlasEntityDef columnClsDef = AtlasTypeUtil .createClassTypeDef(COLUMN_TYPE_V2, null, @@ -379,7 +454,12 @@ public abstract class BaseResourceIT { AtlasTypeUtil.createOptionalAttrDef("createTime", "long"), AtlasTypeUtil.createOptionalAttrDef("lastAccessTime", "date"), AtlasTypeUtil.createOptionalAttrDef("temporary", "boolean"), - AtlasTypeUtil.createRequiredAttrDef("db", DATABASE_TYPE_V2), + new AtlasAttributeDef("db", + DATABASE_TYPE_V2, + true, + Cardinality.SINGLE, + 0, 1, false, true, Collections.singletonList(isCompositeTargetConstraint)), + //some tests don't set the columns field or set it to null... AtlasTypeUtil.createOptionalAttrDef("columns", DataTypes.arrayTypeName(COLUMN_TYPE_V2)), AtlasTypeUtil.createOptionalAttrDef("tableType", "tableType"), @@ -418,7 +498,7 @@ public abstract class BaseResourceIT { ImmutableList.of(classificationTrait, piiTrait, phiTrait, pciTrait, soxTrait, secTrait, financeTrait), ImmutableList.of(dbClsTypeDef, columnClsDef, tblClsDef, loadProcessClsDef)); - createType(typesDef); + batchCreateTypes(typesDef); } AttributeDefinition attrDef(String name, IDataType dT) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5f508c97/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java index 9724c4c..96e6f9b 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -18,16 +18,24 @@ package org.apache.atlas.web.resources; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; -import com.google.inject.Inject; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.core.util.MultivaluedMapImpl; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.fail; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; + import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.model.instance.GuidMapping; import org.apache.atlas.notification.NotificationConsumer; import org.apache.atlas.notification.NotificationInterface; import org.apache.atlas.notification.NotificationModule; @@ -61,18 +69,13 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Guice; import org.testng.annotations.Test; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.fail; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import com.google.inject.Inject; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.core.util.MultivaluedMapImpl; /** @@ -110,6 +113,66 @@ public class EntityJerseyResourceIT extends BaseResourceIT { } @Test + public void testCreateNestedEntities() throws Exception { + + Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); + databaseInstance.set("name", "db1"); + databaseInstance.set("description", "foo database"); + + int nTables = 5; + int colsPerTable=3; + List<Referenceable> tables = new ArrayList<>(); + List<Referenceable> allColumns = new ArrayList<>(); + + for(int i = 0; i < nTables; i++) { + String tableName = "db1-table-" + i; + + Referenceable tableInstance = + new Referenceable(HIVE_TABLE_TYPE); + tableInstance.set("name", tableName); + tableInstance.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName); + tableInstance.set("db", databaseInstance); + tableInstance.set("description", tableName + " table"); + tables.add(tableInstance); + + List<Referenceable> columns = new ArrayList<>(); + for(int j = 0; j < colsPerTable; j++) { + Referenceable columnInstance = new Referenceable(COLUMN_TYPE); + columnInstance.set("name", tableName + "-col-" + j); + columnInstance.set("dataType", "String"); + columnInstance.set("comment", "column " + j + " for table " + i); + allColumns.add(columnInstance); + columns.add(columnInstance); + } + tableInstance.set("columns", columns); + } + + //Create the tables. The database and columns should be created automatically, since + //the tables reference them. + JSONArray entityArray = new JSONArray(tables.size()); + for(int i = 0; i < tables.size(); i++) { + Referenceable table = tables.get(i); + entityArray.put(InstanceSerialization.toJson(table, true)); + } + String json = entityArray.toString(); + + JSONObject response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.CREATE_ENTITY, json); + + GuidMapping guidMapping = GuidMapping.fromString(response.toString()); + + Map<String,String> guidsCreated = guidMapping.getGuidAssignments(); + assertEquals(guidsCreated.size(), nTables * colsPerTable + nTables + 1); + assertNotNull(guidsCreated.get(databaseInstance.getId()._getId())); + for(Referenceable r : allColumns) { + assertNotNull(guidsCreated.get(r.getId()._getId())); + } + for(Referenceable r : tables) { + assertNotNull(guidsCreated.get(r.getId()._getId())); + } + } + + + @Test public void testSubmitEntity() throws Exception { tableInstance = createHiveTableInstanceBuiltIn(DATABASE_NAME, TABLE_NAME, dbId); tableId = createInstance(tableInstance);
