Repository: incubator-atlas Updated Branches: refs/heads/master 100749b49 -> 4c56c61f8
ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4c56c61f Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4c56c61f Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4c56c61f Branch: refs/heads/master Commit: 4c56c61f85c4c38a1d2bec3088130f2ea5f1e8fe Parents: 100749b Author: Shwetha GS <[email protected]> Authored: Thu Sep 29 11:50:47 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Thu Sep 29 11:50:47 2016 +0530 ---------------------------------------------------------------------- release-log.txt | 1 + .../atlas/services/ReservedTypesRegistrar.java | 55 ++++++++++++++++---- .../test/java/org/apache/atlas/TestUtils.java | 49 +++++++++++++++++ .../services/ReservedTypesRegistrarTest.java | 32 ++++++------ 4 files changed, 112 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 0d78273..1a691ca 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via shwethags) ATLAS-1199 Atlas UI not loading after fresh build due to jquery-asBreadcrumbs plugin upgrade (kevalbhatt via shwethags) ATLAS-1174 Framework to apply updates to types in the type-system ([email protected] via shwethags) ATLAS-1155 Errors in Eclipse when I bring in the latest code (davidrad via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java index 41c0155..6e0b586 100644 --- a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java +++ b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java @@ -18,12 +18,12 @@ package org.apache.atlas.services; +import com.google.common.collect.ImmutableList; import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.TypesSerialization; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.*; +import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,8 +31,10 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.List; public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar { @@ -81,13 +83,46 @@ public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar { LOG.error("Error while deserializing JSON in {}", typeDefName); throw new ReservedTypesRegistrationException("Error while deserializing JSON in " + typeDefName, e); } - HierarchicalTypeDefinition<ClassType> classDef = typesDef.classTypesAsJavaList().get(0); - if (!typeSystem.isRegistered(classDef.typeName)) { - metadataService.createType(typeDefJSON); - LOG.info("Registered types in {}", typeDefName); - } else { - LOG.warn("class {} already registered, ignoring types in {}", classDef.typeName, - typeDefName); + List<HierarchicalTypeDefinition<ClassType>> createClassDefList = new ArrayList<>(); + List<HierarchicalTypeDefinition<TraitType>> createTraitDefList = new ArrayList<>(); + List<EnumTypeDefinition> createEnumDefList = new ArrayList<>(); + List<StructTypeDefinition> createStructDefList = new ArrayList<>(); + + for(HierarchicalTypeDefinition<ClassType> classTypeDef:typesDef.classTypesAsJavaList()){ + if(!typeSystem.isRegistered(classTypeDef.typeName)){ + LOG.debug("ClassType {} is not registered. Adding to create type list", classTypeDef.typeName); + createClassDefList.add(classTypeDef); + } + } + + for(HierarchicalTypeDefinition<TraitType> traitTypeDef:typesDef.traitTypesAsJavaList()){ + if(!typeSystem.isRegistered(traitTypeDef.typeName)){ + LOG.debug("TraitType {} is not registered. Adding to create type list", traitTypeDef.typeName); + createTraitDefList.add(traitTypeDef); + } + } + + for(StructTypeDefinition structTypeDef:typesDef.structTypesAsJavaList()){ + if(!typeSystem.isRegistered(structTypeDef.typeName)){ + LOG.debug("StructType {} is not registered. Adding to create type list", structTypeDef.typeName); + createStructDefList.add(structTypeDef); + } + } + + for(EnumTypeDefinition enumTypeDef:typesDef.enumTypesAsJavaList()){ + if(!typeSystem.isRegistered(enumTypeDef.name)){ + LOG.debug("EnumType {} is not registered. Adding to create type list", enumTypeDef.name); + createEnumDefList.add(enumTypeDef); + } + } + + TypesDef createTypes = TypesUtil.getTypesDef(ImmutableList.copyOf(createEnumDefList), ImmutableList.copyOf(createStructDefList), + ImmutableList.copyOf(createTraitDefList), ImmutableList.copyOf(createClassDefList)); + + String createTypeJSON = TypesSerialization.toJson(createTypes); + if(createTypeJSON != null) { + metadataService.createType(createTypeJSON); + LOG.info("Created types definition JSON {}", createTypeJSON); } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index bd9df62..b27854e 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -239,6 +239,55 @@ public final class TestUtils { public static final String NAME = "name"; + public static TypesDef simpleType(){ + HierarchicalTypeDefinition<ClassType> superTypeDefinition = + createClassTypeDef("h_type", ImmutableSet.<String>of(), + createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); + + StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType", + new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)}); + + HierarchicalTypeDefinition<TraitType> traitTypeDefinition = + createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); + + EnumValue values[] = {new EnumValue("ONE", 1),}; + + EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values); + return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), + ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition)); + } + + public static TypesDef simpleTypeUpdated(){ + HierarchicalTypeDefinition<ClassType> superTypeDefinition = + createClassTypeDef("h_type", ImmutableSet.<String>of(), + createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); + + HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition = + createClassTypeDef("new_h_type", ImmutableSet.<String>of(), + createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); + + StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType", + new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)}); + + HierarchicalTypeDefinition<TraitType> traitTypeDefinition = + createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); + + EnumValue values[] = {new EnumValue("ONE", 1),}; + + EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values); + return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), + ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition, newSuperTypeDefinition)); + } + + public static TypesDef simpleTypeUpdatedDiff() { + HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition = + createClassTypeDef("new_h_type", ImmutableSet.<String>of(), + createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); + + return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(newSuperTypeDefinition)); + } + public static TypesDef defineHiveTypes() { String _description = "_description"; HierarchicalTypeDefinition<ClassType> superTypeDefinition = http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java index d602bd8..effab15 100644 --- a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java +++ b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java @@ -23,6 +23,8 @@ import org.apache.atlas.TestUtils; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -55,17 +57,6 @@ public class ReservedTypesRegistrarTest { } @Test - public void testRegisterFirstChecksClassTypeIsRegistered() throws AtlasException { - ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar(); - TypesDef typesDef = TestUtils.defineHiveTypes(); - String typesJson = TypesSerialization.toJson(typesDef); - reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson); - InOrder inOrder = inOrder(typeSystem, metadataService); - inOrder.verify(typeSystem).isRegistered(typesDef.classTypesAsJavaList().get(0).typeName); - inOrder.verify(metadataService).createType(typesJson); - } - - @Test public void testRegisterCreatesTypesUsingMetadataService() throws AtlasException { ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar(); TypesDef typesDef = TestUtils.defineHiveTypes(); @@ -90,12 +81,23 @@ public class ReservedTypesRegistrarTest { } @Test - public void testShouldNotRegisterIfTypeIsAlreadyRegistered() throws AtlasException { + public void testCreateAndUpdateType() throws AtlasException{ ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar(); - TypesDef typesDef = TestUtils.defineHiveTypes(); + TypesDef typesDef = TestUtils.simpleType(); String typesJson = TypesSerialization.toJson(typesDef); - when(typeSystem.isRegistered(typesDef.classTypesAsJavaList().get(0).typeName)).thenReturn(true); reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson); - verifyZeroInteractions(metadataService); + verify(metadataService).createType(typesJson); + + //test update simple type + TypesDef updatedTypesDef = TestUtils.simpleTypeUpdated(); + String updatedTypesJson = TypesSerialization.toJson(updatedTypesDef); + TypesDef simpleTypeUpdatedDiff = TestUtils.simpleTypeUpdatedDiff(); + String simpleTypeUpdatedDiffJson = TypesSerialization.toJson(simpleTypeUpdatedDiff); + when(typeSystem.isRegistered("h_type")).thenReturn(true); + when(typeSystem.isRegistered("t_type")).thenReturn(true); + when(typeSystem.isRegistered("s_type")).thenReturn(true); + when(typeSystem.isRegistered("e_type")).thenReturn(true); + reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", updatedTypesJson); + verify(metadataService).createType(simpleTypeUpdatedDiffJson); } }
