ATLAS-2748, ATLAS-2849, ATLAS-2750: Edges' state preserved during import. Classification dissociation fails on imported entities with classifications. Processing classifications with attributes containing maps and arrays.
(cherry picked from commit 7174257bf131f531a5f86ccb574ddb512af69623) Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/6acbb026 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/6acbb026 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/6acbb026 Branch: refs/heads/branch-1.0 Commit: 6acbb0264926bcdf26b354fbd11132b0d6410a1d Parents: 0e43879 Author: Ashutosh Mestry <ames...@hortonworks.com> Authored: Fri Jun 8 10:35:27 2018 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Sun Jun 10 08:05:58 2018 -0700 ---------------------------------------------------------------------- .../janus/migration/ElementProcessors.java | 4 +- .../migration/TypesWithCollectionsFinder.java | 1 + .../janus/migration/GraphSONUtilityTest.java | 46 +++++++++----------- .../TypesWithCollectionsFinderTest.java | 9 +++- .../json/typesDef-classification-with-map.json | 30 +++++++++++++ 5 files changed, 62 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java index 4017aaa..18082d7 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java @@ -34,6 +34,7 @@ import java.util.UUID; import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID; import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY; @@ -354,9 +355,8 @@ public class ElementProcessors { private void addMandatoryRelationshipProperties(String label, Map<String, Object> props) { props.put(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, UUID.randomUUID().toString()); - props.put(RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, String.valueOf(getDefaultPropagateValue(label))); - props.put(STATE_PROPERTY_KEY, "ACTIVE"); + props.put(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY, false); } } http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java index 55aa9c9..9b4499b 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java @@ -47,6 +47,7 @@ public class TypesWithCollectionsFinder { addVertexPropertiesForCollectionAttributes(typeRegistry.getAllEntityTypes(), ret); addVertexPropertiesForCollectionAttributes(typeRegistry.getAllStructTypes(), ret); + addVertexPropertiesForCollectionAttributes(typeRegistry.getAllClassificationTypes(), ret); displayInfo("types with properties: ", ret); http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java index 0c9b620..3184236 100644 --- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java @@ -38,6 +38,7 @@ import java.util.Set; import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID; import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY; import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY; @@ -229,7 +230,6 @@ public class GraphSONUtilityTest extends BaseUtils { @Test public void tagAssociated_NewAttributesAdded() { - ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap(), new HashMap<>()); TinkerGraph tg = TinkerGraph.open(); GraphSONUtility gu = new GraphSONUtility(elementProcessors); @@ -238,22 +238,8 @@ public class GraphSONUtilityTest extends BaseUtils { addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag()); Iterator<Vertex> vertices = tg.vertices(); - while(vertices.hasNext()) { - Vertex v = vertices.next(); - if(v.id().toString() != "16752") continue; - - assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent()); - assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent()); - assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE"); - } - - Iterator<Edge> edges = tg.edges(); - while(edges.hasNext()) { - Edge e = edges.next(); - assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent()); - assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE"); - assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent()); - } + assertVertex(vertices, "16752"); + assertEdge(tg); } @Test @@ -266,21 +252,31 @@ public class GraphSONUtilityTest extends BaseUtils { addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag()); Iterator<Vertex> vertices = tg.vertices(); - while(vertices.hasNext()) { - Vertex v = vertices.next(); - if(v.id().toString() != "16752") continue; - - assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent()); - assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent()); - assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE"); - } + assertVertex(vertices, "16752"); + assertEdge(tg); + } + private void assertEdge(TinkerGraph tg) { Iterator<Edge> edges = tg.edges(); while(edges.hasNext()) { Edge e = edges.next(); assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent()); + assertTrue(e.property(Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY).isPresent()); assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE"); assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent()); + assertTrue(e.property(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY).isPresent()); + assertFalse((boolean) e.property(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY).value()); + } + } + + private void assertVertex(Iterator<Vertex> vertices, String vertexId) { + while(vertices.hasNext()) { + Vertex v = vertices.next(); + if(v.id().toString() != vertexId) continue; + + assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent()); + assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent()); + assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE"); } } http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java index b4d76db..dad5e4a 100644 --- a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java @@ -20,10 +20,13 @@ package org.apache.atlas.repository.migration; import com.google.inject.Inject; import org.apache.atlas.TestModules; +import org.apache.atlas.TestUtilsV2; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.GraphDBMigrator; import org.apache.atlas.repository.graphdb.janus.migration.TypesWithCollectionsFinder; +import org.apache.atlas.utils.TestResourceFileUtils; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -45,13 +48,15 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts { @BeforeClass public void setup() throws IOException, AtlasBaseException { loadTypesFromJson(); + + typeDefStore.createTypesDef(TestResourceFileUtils.readObjectFromJson(".", "typesDef-classification-with-map", AtlasTypesDef.class)); } @Test public void fetchAll() { Map<String, Map<String, List<String>>> typeAttrMap = TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry); - assertEquals(typeAttrMap.size(), 9); + assertEquals(typeAttrMap.size(), 10); assertProperties(typeAttrMap, "__AtlasUserProfile", "ARRAY", "__AtlasUserProfile.savedSearches"); @@ -75,6 +80,8 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts { assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.partitionKeys"); assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.columns"); assertProperties(typeAttrMap, "hive_table", "MAP_PRIMITIVE", "hive_table.parameters"); + + assertProperties(typeAttrMap, "tag_with_map_of_map", "MAP_PRIMITIVE", "tag_with_map_of_map.tag_with_map_of_map"); } private void assertProperties(Map<String, Map<String, List<String>>> typeAttrMap, String typeName, String typeCategory, String propertyName) { http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/repository/src/test/resources/json/typesDef-classification-with-map.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/typesDef-classification-with-map.json b/repository/src/test/resources/json/typesDef-classification-with-map.json new file mode 100644 index 0000000..cd67df4 --- /dev/null +++ b/repository/src/test/resources/json/typesDef-classification-with-map.json @@ -0,0 +1,30 @@ +{ + "classificationDefs": [ + { + "category": "CLASSIFICATION", + "guid": "d154cc2e-cd67-4620-9893-0cd6aec54244", + "createdBy": "admin", + "updatedBy": "admin", + "createTime": 1528460320484, + "updateTime": 1528460320484, + "version": 1, + "name": "tag_with_map_of_map", + "description": "tag_with_map_of_map", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "tag_with_map_of_map", + "typeName": "map<string,map<string,string>>", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 0, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false + } + ], + "superTypes": [], + "subTypes": [] + } + ] +}