Repository: incubator-atlas Updated Branches: refs/heads/master 5a0be8057 -> e4aa387fb
ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/e4aa387f Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/e4aa387f Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/e4aa387f Branch: refs/heads/master Commit: e4aa387fb497285097e09ee6bac156ba2331f854 Parents: 5a0be80 Author: Suma Shivaprasad <[email protected]> Authored: Fri Apr 1 15:33:51 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Fri Apr 1 15:33:51 2016 -0700 ---------------------------------------------------------------------- release-log.txt | 1 + .../atlas/repository/memory/MemRepository.java | 29 +------ .../typesystem/types/HierarchicalType.java | 20 ++--- .../types/HierarchicalTypeDependencySorter.java | 75 ++++++++++++++++++ .../atlas/typesystem/types/TypeSystem.java | 6 +- .../HierarchicalTypeDependencySorterTest.java | 81 ++++++++++++++++++++ .../atlas/typesystem/types/TypeSystemTest.java | 54 ++++++++----- 7 files changed, 201 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e4aa387f/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 231f0ce..607ff6c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai) ATLAS-525 Drop support for partitions, select query lineage, roles, principals, resource, hive_type...(sumasai via shwethags) ATLAS-599 HDFS Path Model (sumasai via yhemanth) ATLAS-553 Entity mutation - Fix issue with reordering of elements in array<class> with composite references (sumasai via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e4aa387f/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java b/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java index 4cf0a72..75b4979 100755 --- a/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java @@ -30,13 +30,13 @@ import org.apache.atlas.typesystem.persistence.ReferenceableInstance; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.HierarchicalType; +import org.apache.atlas.typesystem.types.HierarchicalTypeDependencySorter; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.ObjectGraphWalker; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,11 +46,6 @@ import java.util.concurrent.atomic.AtomicInteger; public class MemRepository implements IRepository { -/* - public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - public static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd"); -*/ - final TypeSystem typeSystem; /* * A Store for each Class and Trait. @@ -63,23 +58,6 @@ public class MemRepository implements IRepository { this.typeStores = new HashMap<>(); } -/* - @Override - public DateFormat getDateFormat() { - return dateFormat; - } - - @Override - public DateFormat getTimestampFormat() { - return timestampFormat; - } - - @Override - public boolean allowNullsInCollections() { - return false; - } -*/ - @Override public Id newId(String typeName) { return new Id("" + ID_SEQ.incrementAndGet(), 0, typeName); @@ -306,9 +284,8 @@ public class MemRepository implements IRepository { cTypes.add((ClassType) h); } } - - Collections.sort(tTypes); - Collections.sort(cTypes); + tTypes = HierarchicalTypeDependencySorter.sortTypes(tTypes); + cTypes = HierarchicalTypeDependencySorter.sortTypes(cTypes); for (TraitType tT : tTypes) { defineTrait(tT); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e4aa387f/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java index ac32126..89fcea6 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java @@ -46,8 +46,7 @@ import java.util.Set; * as SuperTypes. * @param <T> the class of the Instance of this DataType. */ -public abstract class HierarchicalType<ST extends HierarchicalType, T> extends AbstractDataType<T> - implements Comparable<ST> { +public abstract class HierarchicalType<ST extends HierarchicalType, T> extends AbstractDataType<T> { public final TypeSystem typeSystem; public final Class<ST> superTypeClass; @@ -367,19 +366,10 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A } @Override - public int compareTo(ST o) { - String oName = o.getName(); - try { - if (o.isSubType(getName())) { - return 1; - } else if (isSubType(oName)) { - return -1; - } else { - return getName().compareTo(oName); - } - } catch(AtlasException e) { - throw new RuntimeException(e); - } + public String toString() { + + return "[name=" + name + ", description=" + description + + ", superTypes=" + superTypes + ", immediateAttrs=" + immediateAttrs + "]"; } public Set<String> getAllSuperTypeNames() { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e4aa387f/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorter.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorter.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorter.java new file mode 100644 index 0000000..aaec05c --- /dev/null +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorter.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.typesystem.types; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.google.common.collect.ImmutableSet; + + +/** + * Sorts hierarchical types by supertype dependency + */ +public class HierarchicalTypeDependencySorter { + + /** + * Sorts the specified hierarchical types by supertype dependencies, + * such that any type A which is a supertype of type B + * will always be located earlier in the result list; that is, the supertype + * A would be found at some index i and subtype B would be found at some index j, + * and i < j. + * + * @param types hierarchical types to be sorted + * @return hierarchical types sorted by supertype dependency + */ + public static <T extends HierarchicalType> List<T> sortTypes(List<T> types) { + Map<String, T> typesByName = new HashMap<>(); + for (T type : types) { + typesByName.put(type.name, type); + } + List<T> result = new ArrayList<>(types.size()); + Set<T> processed = new HashSet<>(); + for (T type : types) { + addToResult(type, result, processed, typesByName); + } + return result; + } + + private static <T extends HierarchicalType> void addToResult(T type, List<T> result, + Set<T> processed, Map<String, T> typesByName) { + + if (processed.contains(type)) { + return; + } + processed.add(type); + ImmutableSet<String> superTypeNames = type.superTypes; + for (String superTypeName : superTypeNames) { + // Recursively add any supertypes first to the result. + T superType = typesByName.get(superTypeName); + if (superType != null) { + addToResult(superType, result, processed, typesByName); + } + } + result.add(type); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e4aa387f/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java index 402800e..7763ebc 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java @@ -34,8 +34,6 @@ import javax.inject.Singleton; import java.lang.reflect.Constructor; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -542,13 +540,13 @@ public class TypeSystem { for (String traitTypeName : traitNameToDefMap.keySet()) { traitTypes.add(getDataType(TraitType.class, traitTypeName)); } - Collections.sort(traitTypes); + traitTypes = HierarchicalTypeDependencySorter.sortTypes(traitTypes); List<ClassType> classTypes = new ArrayList<>(); for (String classTypeName : classNameToDefMap.keySet()) { classTypes.add(getDataType(ClassType.class, classTypeName)); } - Collections.sort(classTypes); + classTypes = HierarchicalTypeDependencySorter.sortTypes(classTypes); for (StructTypeDefinition structDef : structDefs) { constructStructureType(structDef); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e4aa387f/typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorterTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorterTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorterTest.java new file mode 100644 index 0000000..19bdccf --- /dev/null +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeDependencySorterTest.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.typesystem.types; + +import java.util.Arrays; +import java.util.List; + +import org.apache.atlas.AtlasException; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableSet; + + +public class HierarchicalTypeDependencySorterTest { + + @BeforeMethod + public void setup() throws Exception { + TypeSystem ts = TypeSystem.getInstance(); + ts.reset(); + } + + @SuppressWarnings("rawtypes") + @Test + public void testSimpleModel() throws AtlasException { + TypeSystem ts = TypeSystem.getInstance(); + HierarchicalType a = new ClassType(ts, "a", null, ImmutableSet.<String>of(), 0); + HierarchicalType b = new ClassType(ts, "B", null, ImmutableSet.of("a"), 0); + HierarchicalType c = new ClassType(ts, "C", null, ImmutableSet.of("B"), 0); + + List<HierarchicalType> unsortedTypes = Arrays.asList(c, a, b); + List<HierarchicalType> sortedTypes = HierarchicalTypeDependencySorter.sortTypes(unsortedTypes); + Assert.assertEquals(sortedTypes.size(), 3); + Assert.assertEquals(sortedTypes.indexOf(a), 0); + Assert.assertEquals(sortedTypes.indexOf(b), 1); + Assert.assertEquals(sortedTypes.indexOf(c), 2); + } + + @SuppressWarnings("rawtypes") + @Test + public void testLargerModel() throws Exception { + TypeSystem ts = TypeSystem.getInstance(); + HierarchicalType testObjectType = new ClassType(ts, "TestObject", null, ImmutableSet.<String>of(), 0); + HierarchicalType testDataSetType = new ClassType(ts, "TestDataSet", null, ImmutableSet.of("TestObject"), 0); + HierarchicalType testColumnType = new ClassType(ts, "TestColumn", null, ImmutableSet.of("TestObject"), 0); + HierarchicalType testRelationalDataSetType = new ClassType(ts, "TestRelationalDataSet", null, ImmutableSet.of("TestDataSet"), 0); + HierarchicalType testTableType = new ClassType(ts, "Table", null, ImmutableSet.of("TestDataSet"), 0); + HierarchicalType testDataFileType = new ClassType(ts, "TestDataFile", null, ImmutableSet.of("TestRelationalDataSet"), 0); + HierarchicalType testDocumentType = new ClassType(ts, "TestDocument", null, ImmutableSet.of("TestDataSet"), 0); + HierarchicalType testAnnotationType = new ClassType(ts, "TestAnnotation", null, ImmutableSet.<String>of(), 0); + HierarchicalType myNewAnnotationType = new ClassType(ts, "MyNewAnnotation", null, ImmutableSet.of("TestAnnotation"), 0); + + List<HierarchicalType> unsortedTypes = Arrays.asList(testTableType, testColumnType, myNewAnnotationType, testDataSetType, + testDataFileType, testAnnotationType, testRelationalDataSetType, testObjectType, testDocumentType); + List<HierarchicalType> sortedTypes = HierarchicalTypeDependencySorter.sortTypes(unsortedTypes); + // Verify that super types were sorted before their subtypes. + Assert.assertTrue(sortedTypes.indexOf(testObjectType) < sortedTypes.indexOf(testDataSetType)); + Assert.assertTrue(sortedTypes.indexOf(testObjectType) < sortedTypes.indexOf(testColumnType)); + Assert.assertTrue(sortedTypes.indexOf(testDataSetType) < sortedTypes.indexOf(testRelationalDataSetType)); + Assert.assertTrue(sortedTypes.indexOf(testDataSetType) < sortedTypes.indexOf(testDocumentType)); + Assert.assertTrue(sortedTypes.indexOf(testDataSetType) < sortedTypes.indexOf(testTableType)); + Assert.assertTrue(sortedTypes.indexOf(testRelationalDataSetType) < sortedTypes.indexOf(testDataFileType)); + Assert.assertTrue(sortedTypes.indexOf(testAnnotationType) < sortedTypes.indexOf(myNewAnnotationType)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e4aa387f/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java index a3be4c5..c0574ce 100755 --- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java @@ -36,6 +36,7 @@ import java.util.HashMap; import java.util.List; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef; @@ -171,25 +172,6 @@ public class TypeSystemTest extends BaseTest { } @Test - public void testHierarchy() throws AtlasException { - HierarchicalTypeDefinition<ClassType> a = TypesUtil.createClassTypeDef("a", ImmutableSet.<String>of()); - HierarchicalTypeDefinition<ClassType> b = TypesUtil.createClassTypeDef("B", ImmutableSet.of("a")); - HierarchicalTypeDefinition<ClassType> c = TypesUtil.createClassTypeDef("C", ImmutableSet.of("B")); - - TypeSystem ts = getTypeSystem(); - ts.defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.of(a, b, c)); - ClassType ac = ts.getDataType(ClassType.class, "a"); - ClassType bc = ts.getDataType(ClassType.class, "B"); - ClassType cc = ts.getDataType(ClassType.class, "C"); - - assertTrue(ac.compareTo(bc) < 0); - assertTrue(bc.compareTo(cc) < 0); - assertTrue(ac.compareTo(cc) < 0); - } - - @Test public void testTypeCategory() throws AtlasException { TypeSystem ts = getTypeSystem(); ts.reset(); @@ -251,5 +233,37 @@ public class TypeSystemTest extends BaseTest { Assert.assertEquals(traitNames.size(), numTraits+2); } - + @Test + public void testHierarchy() throws Exception { + HierarchicalTypeDefinition<ClassType> testObjectDef = TypesUtil.createClassTypeDef("TestObject", ImmutableSet.<String>of(), + createOptionalAttrDef("name", DataTypes.STRING_TYPE), + createOptionalAttrDef("description", DataTypes.STRING_TYPE), + createOptionalAttrDef("topAttribute", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> testDataSetDef = TypesUtil.createClassTypeDef("TestDataSet", ImmutableSet.of("TestObject")); + HierarchicalTypeDefinition<ClassType> testColumnDef = TypesUtil.createClassTypeDef("TestColumn", ImmutableSet.of("TestObject"), + createRequiredAttrDef("name", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> testRelationalDataSetDef = + TypesUtil.createClassTypeDef("TestRelationalDataSet", ImmutableSet.of("TestDataSet"), + new AttributeDefinition("columns", DataTypes.arrayTypeName("TestColumn"), + Multiplicity.OPTIONAL, true, null)); + HierarchicalTypeDefinition<ClassType> testTableDef = TypesUtil.createClassTypeDef("TestTable", ImmutableSet.of("TestRelationalDataSet"), + createOptionalAttrDef("schema", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> testDataFileDef = TypesUtil.createClassTypeDef("TestDataFile", ImmutableSet.of("TestRelationalDataSet"), + createOptionalAttrDef("urlString", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> testDocumentDef = TypesUtil.createClassTypeDef("TestDocument", ImmutableSet.of("TestDataSet"), + createOptionalAttrDef("urlString", DataTypes.STRING_TYPE), + createOptionalAttrDef("encoding", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> testAnnotationDef =TypesUtil.createClassTypeDef("TestAnnotation", ImmutableSet.<String>of(), + createOptionalAttrDef("inheritedAttribute", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> myNewAnnotationDef = TypesUtil.createClassTypeDef("MyNewAnnotation", ImmutableSet.of("TestAnnotation"), + createRequiredAttrDef("myNewAnnotationAttribute", DataTypes.STRING_TYPE)); + getTypeSystem().defineTypes(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(testObjectDef, testDataSetDef, testColumnDef, testRelationalDataSetDef, testTableDef, testDataFileDef, testDocumentDef, testAnnotationDef, myNewAnnotationDef)); + + // Verify that field mappings for MyNewAnnotation contains the attribute inherited from the TestAnnotation superclass. + // Prior to fix for ATLAS-573, the inherited attribute was missing. + ClassType dataType = getTypeSystem().getDataType(ClassType.class, "MyNewAnnotation"); + Assert.assertTrue(dataType.fieldMapping.fields.containsKey("inheritedAttribute")); + } }
