Repository: incubator-atlas Updated Branches: refs/heads/master 919120f65 -> bf5672c54
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java new file mode 100644 index 0000000..4a6ed2d --- /dev/null +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeUpdateBaseTest.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.typesystem.types; + +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.testng.Assert; + +public abstract class TypeUpdateBaseTest extends BaseTest { + protected void testTypeUpdateForAttributes() throws Exception { + StructTypeDefinition typeDefinition = + getTypeDefinition(newName(), TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE)); + TypeSystem ts = getTypeSystem(); + TypesDef typesDef = getTypesDef(typeDefinition); + ts.defineTypes(typesDef); + String typeName = typeDefinition.typeName; + + //Allow modifying required to optional attribute + typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE)); + ts.updateTypes(getTypesDef(typeDefinition)); + + //Allow adding new optional attribute + typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE), + TypesUtil.createOptionalAttrDef("b", DataTypes.INT_TYPE)); + ts.updateTypes(getTypesDef(typeDefinition)); + + //Don't allow adding required attribute + typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE), + TypesUtil.createOptionalAttrDef("b", DataTypes.INT_TYPE), + TypesUtil.createRequiredAttrDef("c", DataTypes.INT_TYPE)); + try { + ts.updateTypes(getTypesDef(typeDefinition)); + Assert.fail("Expected TypeUpdateException"); + } catch (TypeUpdateException e) { + //assert that type is not updated when validation fails + Assert.assertEquals(getNumberOfFields(ts, typeDefinition.typeName), 2); + } + + //Don't allow removing attribute + typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE)); + try { + ts.updateTypes(getTypesDef(typeDefinition)); + } catch (TypeUpdateException e) { + //expected + } + + //Don't allow modifying other fields of attribute definition - optional to required + typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE), + TypesUtil.createRequiredAttrDef("b", DataTypes.INT_TYPE)); + try { + ts.updateTypes(getTypesDef(typeDefinition)); + } catch (TypeUpdateException e) { + //expected + } + + //Don't allow modifying other fields of attribute definition - attribute type change + typeDefinition = getTypeDefinition(typeName, TypesUtil.createOptionalAttrDef("a", DataTypes.INT_TYPE), + TypesUtil.createOptionalAttrDef("b", DataTypes.STRING_TYPE)); + try { + ts.updateTypes(getTypesDef(typeDefinition)); + } catch (TypeUpdateException e) { + //expected + } + + //Don't allow modifying other fields of attribute definition - attribute type change + typeDefinition = getTypeDefinition(typeName, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE), + new AttributeDefinition("b", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()), + Multiplicity.COLLECTION, false, null)); + try { + ts.updateTypes(getTypesDef(typeDefinition)); + } catch (TypeUpdateException e) { + //expected + } + } + + protected abstract int getNumberOfFields(TypeSystem ts, String typeName) throws Exception; + + protected abstract TypesDef getTypesDef(StructTypeDefinition typeDefinition); + + protected abstract StructTypeDefinition getTypeDefinition(String typeName, + AttributeDefinition... attributeDefinitions); +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java index 7b8f88c..1251062 100755 --- a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java +++ b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java @@ -35,7 +35,6 @@ import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.codehaus.jettison.json.JSONArray; @@ -153,7 +152,7 @@ public class QuickStart { HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null); - return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef), ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef)); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java index c049350..1f0b98a 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java @@ -38,6 +38,7 @@ import javax.ws.rs.Consumes; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -110,6 +111,50 @@ public class TypesResource { } /** + * Update of existing types - if the given type doesn't exist, creates new type + * Allowed updates are: + * 1. Add optional attribute + * 2. Change required to optional attribute + * 3. Add super types - super types shouldn't contain any required attributes + * @param request + * @return + */ + @PUT + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response update(@Context HttpServletRequest request) { + try { + final String typeDefinition = Servlets.getRequestPayload(request); + LOG.debug("Updating type with definition {} ", typeDefinition); + + JSONObject typesJson = metadataService.updateType(typeDefinition); + final JSONArray typesJsonArray = typesJson.getJSONArray(AtlasClient.TYPES); + + JSONArray typesResponse = new JSONArray(); + for (int i = 0; i < typesJsonArray.length(); i++) { + final String name = typesJsonArray.getString(i); + typesResponse.put(new JSONObject() {{ + put(AtlasClient.NAME, name); + }}); + } + + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); + response.put(AtlasClient.TYPES, typesResponse); + return Response.ok().entity(response).build(); + } catch (TypeExistsException e) { + LOG.error("Type already exists", e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT)); + } catch (AtlasException | IllegalArgumentException e) { + LOG.error("Unable to persist types", e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to persist types", e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** * Fetch the complete definition of a given type name which is unique. * * @param typeName name of a type which is unique. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java index 6976c45..291ef48 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java @@ -41,7 +41,6 @@ import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.web.util.Servlets; import org.apache.commons.configuration.Configuration; @@ -188,7 +187,7 @@ public abstract class BaseResourceIT { HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL", null); - TypesDef typesDef = TypeUtils.getTypesDef(ImmutableList.of(enumTypeDefinition), + TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), ImmutableList.of(classificationTrait, piiTrait, phiTrait, pciTrait, soxTrait, secTrait, financeTrait, dimTraitDef, factTraitDef, metricTraitDef, etlTraitDef), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java index 8053c9f..380f280 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -38,7 +38,6 @@ import org.apache.atlas.typesystem.types.EnumTypeDefinition; import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.web.util.Servlets; import org.apache.commons.lang.RandomStringUtils; @@ -112,6 +111,34 @@ public class EntityJerseyResourceIT extends BaseResourceIT { Assert.assertNotNull(response.get(AtlasClient.GUID)); } + @Test + public void testEntityDefinitionAcrossTypeUpdate() throws Exception { + //create type + HierarchicalTypeDefinition<ClassType> typeDefinition = TypesUtil + .createClassTypeDef(randomString(), ImmutableList.<String>of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE)); + serviceClient.createType(TypesSerialization.toJson(typeDefinition, false)); + + //create entity for the type + Referenceable instance = new Referenceable(typeDefinition.typeName); + instance.set("name", randomString()); + String guid = serviceClient.createEntity(instance).getString(0); + + //update type - add attribute + typeDefinition = TypesUtil.createClassTypeDef(typeDefinition.typeName, ImmutableList.<String>of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE)); + TypesDef typeDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), + ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(typeDefinition)); + serviceClient.updateType(typeDef); + + //Get definition after type update - new attributes should be null + Referenceable entity = serviceClient.getEntity(guid); + Assert.assertNull(entity.get("description")); + Assert.assertEquals(entity.get("name"), instance.get("name")); + } + @DataProvider public Object[][] invalidAttrValues() { return new Object[][]{{null}, {""}}; @@ -506,10 +533,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT { HierarchicalTypeDefinition<ClassType> classTypeDefinition = TypesUtil .createClassTypeDef(classType, ImmutableList.<String>of(), TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE)); - TypesDef typesDef = TypeUtils - .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.of(classTypeDefinition)); + TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(classTypeDefinition)); createType(typesDef); Referenceable instance = new Referenceable(classType); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java index 84036bd..d255b75 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java @@ -32,7 +32,6 @@ import org.apache.atlas.typesystem.types.EnumTypeDefinition; import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.web.util.Servlets; import org.codehaus.jettison.json.JSONArray; @@ -176,8 +175,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT { HierarchicalTypeDefinition<TraitType> financeTrait = TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of()); - TypesDef typesDef = TypeUtils - .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), ImmutableList .of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait, soxTrait, secTrait, financeTrait), ImmutableList.of(dslTestTypeDefinition)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java index a9e8eba..ac1b5ca 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java @@ -28,8 +28,10 @@ import org.apache.atlas.typesystem.json.TypesSerialization$; import org.apache.atlas.typesystem.types.AttributeDefinition; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.web.util.Servlets; @@ -87,6 +89,33 @@ public class TypesJerseyResourceIT extends BaseResourceIT { } } + @Test + public void testUpdate() throws Exception { + HierarchicalTypeDefinition<ClassType> typeDefinition = TypesUtil + .createClassTypeDef(randomString(), ImmutableList.<String>of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE)); + List<String> typesCreated = serviceClient.createType(TypesSerialization.toJson(typeDefinition, false)); + Assert.assertEquals(typesCreated.size(), 1); + Assert.assertEquals(typesCreated.get(0), typeDefinition.typeName); + + //Add super type + HierarchicalTypeDefinition<ClassType> superTypeDefinition = TypesUtil + .createClassTypeDef(randomString(), ImmutableList.<String>of(), + TypesUtil.createOptionalAttrDef("sname", DataTypes.STRING_TYPE)); + + typeDefinition = TypesUtil.createClassTypeDef(typeDefinition.typeName, + ImmutableList.of(superTypeDefinition.typeName), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE)); + TypesDef typeDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), + ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(superTypeDefinition, typeDefinition)); + List<String> typesUpdated = serviceClient.updateType(typeDef); + Assert.assertEquals(typesUpdated.size(), 2); + Assert.assertTrue(typesUpdated.contains(superTypeDefinition.typeName)); + Assert.assertTrue(typesUpdated.contains(typeDefinition.typeName)); + } + @Test(dependsOnMethods = "testSubmit") public void testGetDefinition() throws Exception { for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
