http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java index e863d9f..aee1b69 100644 --- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java @@ -26,14 +26,16 @@ import org.apache.tinkerpop.gremlin.structure.io.graphson.TypeInfo; import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; import org.apache.tinkerpop.shaded.jackson.databind.JsonNode; import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper; -import org.testng.ITestContext; +import org.testng.SkipException; import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; import java.io.File; import java.io.IOException; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.testng.AssertJUnit.assertTrue; @@ -41,12 +43,16 @@ public class BaseUtils { private static final String resourcesDirRelativePath = "/src/test/resources/"; private String resourceDir; - protected final RelationshipTypeCache emptyRelationshipCache = new RelationshipTypeCache(new HashMap<>()); + protected final ElementProcessors emptyRelationshipCache = new ElementProcessors(new HashMap<>(), new HashMap<>()); protected GraphSONUtility graphSONUtility; - protected Object[][] getJsonNodeFromFile(String s) throws IOException { + protected JsonNode getJsonNodeFromFile(String s) { File f = new File(getFilePath(s)); - return new Object[][]{{getEntityNode(FileUtils.readFileToString(f))}}; + try { + return getEntityNode(FileUtils.readFileToString(f)); + } catch (IOException e) { + throw new SkipException("getJsonNodeFromFile: " + s, e); + } } protected String getFilePath(String fileName) { @@ -76,12 +82,18 @@ public class BaseUtils { utility.vertexFromJson(tg, node); } - protected void addEdge(TinkerGraph tg, MappedElementCache cache) throws IOException { + protected void addEdge(TinkerGraph tg, MappedElementCache cache) { GraphSONUtility gu = graphSONUtility; - gu.vertexFromJson(tg, (JsonNode) (getDBV(null)[0][0])); - gu.vertexFromJson(tg, (JsonNode) (getTableV(null))[0][0]); - gu.edgeFromJson(tg, cache, (JsonNode) getEdge(null)[0][0]); + addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2()); + addEdgeToGraph(tg, gu, cache, getEdge(), getEdgeCol(), getEdgeCol2()); + } + + protected void addEdgesForMap(TinkerGraph tg, MappedElementCache cache) { + GraphSONUtility gu = graphSONUtility; + + addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2()); + addEdgeToGraph(tg, gu, cache, getEdgeCol3(), getEdgeCol4()); } protected Vertex fetchTableVertex(TinkerGraph tg) { @@ -91,29 +103,84 @@ public class BaseUtils { return (Vertex) query.next(); } - @DataProvider(name = "col1") - public Object[][] getCol1(ITestContext context) throws IOException { + protected Map<String, Map<String, List<String>>> getTypePropertyMap(String type, String property, String category) { + Map<String, Map<String, List<String>>> map = new HashMap<>(); + map.put(type, new HashMap<>()); + map.get(type).put(category, new ArrayList<>()); + map.get(type).get(category).add(property); + return map; + } + + protected void addVertexToGraph(TinkerGraph tg, GraphSONUtility gu, JsonNode... nodes) { + for(JsonNode n : nodes) { + gu.vertexFromJson(tg, n); + } + } + + protected void addEdgeToGraph(TinkerGraph tg, GraphSONUtility gu, MappedElementCache cache, JsonNode... nodes) { + + for(JsonNode n : nodes) { + gu.edgeFromJson(tg, cache, n); + } + } + + public JsonNode getCol1() { return getJsonNodeFromFile("col-legacy.json"); } - @DataProvider(name = "dbType") - public Object[][] getDbType(ITestContext context) throws IOException { + public JsonNode getCol2() { + return getJsonNodeFromFile("col-2-legacy.json"); + } + + public JsonNode getCol3() { + return getJsonNodeFromFile("col-3-legacy.json"); + } + + public JsonNode getDbType() { return getJsonNodeFromFile("db-type-legacy.json"); } - @DataProvider(name = "edge") - public Object[][] getEdge(ITestContext context) throws IOException { + public JsonNode getEdge() { return getJsonNodeFromFile("edge-legacy.json"); } - @DataProvider(name = "dbV") - public Object[][] getDBV(ITestContext context) throws IOException { - return getJsonNodeFromFile("db-v-65544.json"); + public JsonNode getEdgeCol() { + return getJsonNodeFromFile("edge-legacy-col.json"); } + public JsonNode getEdgeCol2() { + return getJsonNodeFromFile("edge-legacy-col2.json"); + } + + public JsonNode getEdgeCol3() { + return getJsonNodeFromFile("edge-legacy-col3.json"); + } + + public JsonNode getEdgeCol4() { + return getJsonNodeFromFile("edge-legacy-col4.json"); + } + + public JsonNode getEdgeTag() { + return getJsonNodeFromFile("edge-legacy-tag.json"); + } + + public JsonNode getDBV() { + return getJsonNodeFromFile("db-v-65544.json"); + } - @DataProvider(name = "tableV") - public Object[][] getTableV(ITestContext context) throws IOException { + public JsonNode getTableV() { return getJsonNodeFromFile("table-v-147504.json"); } + + public JsonNode getTagV() { + return getJsonNodeFromFile("tag-163856752.json"); + } + + public JsonNode getProcessV() { + return getJsonNodeFromFile("lineage-v-98312.json"); + } + + public JsonNode getEdgeProcess() { + return getJsonNodeFromFile("edge-legacy-process.json"); + } }
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java deleted file mode 100644 index 4d73c78..0000000 --- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.graphdb.janus.migration; - -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; -import org.eclipse.jetty.util.BlockingArrayQueue; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.BlockingQueue; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.AssertJUnit.assertTrue; - -public class GraphSONUtilityPostProcessTest extends BaseUtils { - final String HIVE_COLUMNS_PROPERTY = "hive_table.columns"; - final String edgeId1 = "816u-35tc-ao0l-47so"; - final String edgeId2 = "82rq-35tc-ao0l-2glc"; - - final String edgeId1x = "816u-35tc-ao0l-xxxx"; - final String edgeId2x = "82rq-35tc-ao0l-xxxx"; - - private TinkerGraph tg; - private MappedElementCache cache = new MappedElementCache(); - private Vertex tableV; - - @Test - public void noRefNoUpdate() throws IOException { - tg = TinkerGraph.open(); - graphSONUtility = new GraphSONUtility(emptyRelationshipCache); - - addEdge(tg, cache); - - tableV = fetchTableVertex(tg); - assertNotNull(tableV); - - assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV); - - graphSONUtility.replaceReferencedEdgeIdForList(tg, cache, tableV, HIVE_COLUMNS_PROPERTY); - assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV); - } - - @Test(dependsOnMethods = "noRefNoUpdate") - public void refFoundVertexUpdated() throws IOException { - - cache.lruEdgeCache.put(edgeId1, edgeId1x); - cache.lruEdgeCache.put(edgeId2, edgeId2x); - - graphSONUtility.replaceReferencedEdgeIdForList(tg, cache, tableV, HIVE_COLUMNS_PROPERTY); - assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1x, edgeId2x, tableV); - } - - @Test(dependsOnMethods = "refFoundVertexUpdated") - public void updateUsingPostProcessConsumer() throws IOException { - MappedElementCache cache = new MappedElementCache(); - BlockingQueue<Object> bc = new BlockingArrayQueue<>(); - PostProcessManager.Consumer consumer = new PostProcessManager.Consumer(bc, tg, graphSONUtility, - new String[] {HIVE_COLUMNS_PROPERTY}, cache, 5); - - cache.lruEdgeCache.put(edgeId1x, edgeId1); - cache.lruEdgeCache.put(edgeId2x, edgeId2); - consumer.processItem(tableV.id()); - - assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV); - } - - private void assertListProperty(String HIVE_COLUMNS_PROPERTY, String edgeId1, String edgeId2, Vertex tableV) { - assertTrue(tableV.property(HIVE_COLUMNS_PROPERTY).isPresent()); - List list = (List) tableV.property(HIVE_COLUMNS_PROPERTY).value(); - - assertEquals(list.size(), 2); - assertEquals(list.get(0), edgeId1); - assertEquals(list.get(1), edgeId2); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java index 794b547..049fd64 100644 --- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java @@ -18,36 +18,58 @@ package org.apache.atlas.repository.graphdb.janus.migration; +import org.apache.atlas.model.typedef.AtlasRelationshipDef; +import org.apache.atlas.repository.Constants; +import org.apache.commons.lang.StringUtils; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; import org.apache.tinkerpop.shaded.jackson.databind.JsonNode; import org.testng.Assert; -import org.testng.ITestContext; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.Set; +import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID; +import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY; import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY; +import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.VERTEX_ID_IN_IMPORT_KEY; import static org.testng.Assert.*; public class GraphSONUtilityTest extends BaseUtils { - @Test(dataProvider = "col1") - public void idFetch(JsonNode node) { + private final String HIVE_TABLE_NAME_PROPERTY = "Asset.name"; + private final String HIVE_TABLE_NAME = "test_table_view"; + private final String HIVE_TABLE_COLUMNS_RELATIONSHIP = "hive_table.columns"; + private final String HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP = "hive_table.columnsMap"; + private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP = "hive_table.parameters"; + private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY = "transient_lastDdlTime"; + private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP_VALUE_KEY = String.format("%s.%s", HIVE_TABLE_COLUMNS_PARAMETERS_MAP, HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY); + private final String HIVE_TABLE_TYPE = "hive_table"; + + @Test + public void idFetch() { + JsonNode node = getCol1(); + final int EXPECTED_ID = 98336; Object o = GraphSONUtility.getTypedValueFromJsonNode(node.get(GraphSONTokensTP2._ID)); assertNotNull(o); - assertEquals((int) o, 98336); + assertEquals((int) o, EXPECTED_ID); } - @Test(dataProvider = "col1") - public void verifyReadProperties(JsonNode node) { + @Test + public void verifyReadProperties() { + JsonNode node = getCol1(); Map<String, Object> props = GraphSONUtility.readProperties(node); assertEquals(props.get("__superTypeNames").getClass(), ArrayList.class); @@ -58,8 +80,9 @@ public class GraphSONUtilityTest extends BaseUtils { assertNotNull(props); } - @Test(dataProvider = "col1") - public void dataNodeReadAndVertexAddedToGraph(JsonNode entityNode) throws IOException { + @Test + public void dataNodeReadAndVertexAddedToGraph() throws IOException { + JsonNode entityNode = getCol1(); TinkerGraph tg = TinkerGraph.open(); GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache); Map<String, Object> map = gu.vertexFromJson(tg, entityNode); @@ -71,8 +94,9 @@ public class GraphSONUtilityTest extends BaseUtils { assertTrue(v.property(VERTEX_ID_IN_IMPORT_KEY).isPresent()); } - @Test(dataProvider = "dbType") - public void typeNodeReadAndVertexNotAddedToGraph(JsonNode entityNode) throws IOException { + @Test + public void typeNodeReadAndVertexNotAddedToGraph() throws IOException { + JsonNode entityNode = getDbType(); TinkerGraph tg = TinkerGraph.open(); GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache); gu.vertexFromJson(tg, entityNode); @@ -81,19 +105,46 @@ public class GraphSONUtilityTest extends BaseUtils { } @Test - public void edgeReadAndAddedToGraph() throws IOException { + public void updateNonPrimitiveArrayProperty() throws IOException { + ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getNonPrimitiveArray()); + TinkerGraph tg = TinkerGraph.open(); - GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache); - Map<String, Object> m = null; + GraphSONUtility gu = new GraphSONUtility(elementProcessors); + addVertexToGraph(tg, gu, getTableV()); - m = gu.vertexFromJson(tg, (JsonNode) (getDBV(null)[0][0])); - assertNull(m); + Vertex v = tg.traversal().V().next(); + assertNotNull(v); + assertTrue(v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).isPresent()); - m = gu.vertexFromJson(tg, (JsonNode) (getTableV(null))[0][0]); - assertNull(m); + Map<String, String> list = (Map<String, String>) v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).value(); + assertEquals(list.size(), 2); + } - m = gu.edgeFromJson(tg, new MappedElementCache(), (JsonNode) getEdge(null)[0][0]); - assertNull(m); + + @Test + public void updatePrimitiveMapProperty() { + ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMapPrimitive()); + + TinkerGraph tg = TinkerGraph.open(); + GraphSONUtility gu = new GraphSONUtility(elementProcessors); + addVertexToGraph(tg, gu, getTableV()); + + Vertex v = tg.traversal().V().next(); + assertNotNull(v); + assertTrue(v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).isPresent()); + assertEquals(((Map) v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).value()).size(), 1); + assertEquals(((Map) v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).value()).get(HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY), "1522693834"); + assertFalse(v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP_VALUE_KEY).isPresent()); + } + + @Test + public void edgeReadAndAddedToGraph() { + TinkerGraph tg = TinkerGraph.open(); + GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache); + Map<String, Object> m = null; + + addVertexToGraph(tg, gu, getDBV(), getTableV()); + addEdgeToGraph(tg, gu, new MappedElementCache(), getEdge()); Assert.assertEquals((long) tg.traversal().V().count().next(), 2L); Assert.assertEquals((long) tg.traversal().E().count().next(), 1L); @@ -101,4 +152,179 @@ public class GraphSONUtilityTest extends BaseUtils { Edge e = tg.edges().next(); assertTrue(e.property(EDGE_ID_IN_IMPORT_KEY).isPresent()); } + + @Test + public void edgeReadAndArrayIndexAdded() throws IOException { + ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap()); + TinkerGraph tg = TinkerGraph.open(); + GraphSONUtility gu = new GraphSONUtility(elementProcessors); + Map<String, Object> m = null; + + addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2()); + addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeCol(), getEdgeCol2()); + + Iterator<Edge> edges = tg.edges(); + while(edges.hasNext()) { + Edge e = edges.next(); + String arraySpecificLabel = StringUtils.remove(e.label(), Constants.INTERNAL_PROPERTY_KEY_PREFIX); + if(arraySpecificLabel.equals(HIVE_TABLE_COLUMNS_RELATIONSHIP)) { + assertTrue(e.property(ATTRIBUTE_INDEX_PROPERTY_KEY).isPresent()); + } + assertTrue(e.property(EDGE_ID_IN_IMPORT_KEY).isPresent()); + } + + Iterator<Vertex> vertices = tg.vertices(); + while(vertices.hasNext()) { + Vertex v = vertices.next(); + if(v.property(HIVE_TABLE_NAME_PROPERTY).isPresent()) { + if(v.property(HIVE_TABLE_NAME_PROPERTY).value().toString().equals(HIVE_TABLE_NAME)) { + assertTrue(v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).isPresent()); + } + } + } + } + + @Test + public void nonPrimitiveMap_Removed() throws IOException { + Set<String> actualKeys = new HashSet<String>() {{ + add("col3"); + add("col4"); + }}; + + ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap()); + TinkerGraph tg = TinkerGraph.open(); + GraphSONUtility gu = new GraphSONUtility(elementProcessors); + + addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2()); + addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeCol3(), getEdgeCol4()); + + Iterator<Vertex> vertices = tg.vertices(); + while(vertices.hasNext()) { + Vertex v = vertices.next(); + if(!v.property(HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP).isPresent()) continue; + + fail("Non-primitive map should be removed during vertex creation."); + } + + Iterator<Edge> edges = tg.edges(); + while(edges.hasNext()) { + Edge e = edges.next(); + String mapSpecificLabel = StringUtils.remove(e.label(), Constants.INTERNAL_PROPERTY_KEY_PREFIX); + assertEquals(mapSpecificLabel, HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP); + assertTrue(e.property(ATTRIBUTE_KEY_PROPERTY_KEY).isPresent()); + + assertTrue(actualKeys.contains((String) e.property(ATTRIBUTE_KEY_PROPERTY_KEY).value())); + } + } + + @Test + public void tagAssociated_NewAttributesAdded() throws IOException { + + ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap()); + TinkerGraph tg = TinkerGraph.open(); + GraphSONUtility gu = new GraphSONUtility(elementProcessors); + + addVertexToGraph(tg, gu, getTagV(), getDBV(), getTableV(), getCol3()); + addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag()); + + Iterator<Vertex> vertices = tg.vertices(); + while(vertices.hasNext()) { + Vertex v = vertices.next(); + if(v.id().toString() != "16752") continue; + + assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent()); + assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent()); + assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE"); + } + + Iterator<Edge> edges = tg.edges(); + while(edges.hasNext()) { + Edge e = edges.next(); + assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent()); + assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE"); + assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent()); + } + } + + @Test + public void processEdge_PropagateSetTo_NONE() throws IOException { + ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap()); + TinkerGraph tg = TinkerGraph.open(); + GraphSONUtility gu = new GraphSONUtility(elementProcessors); + + addVertexToGraph(tg, gu, getTagV(), getDBV(), getTableV(), getCol3()); + addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag()); + + Iterator<Vertex> vertices = tg.vertices(); + while(vertices.hasNext()) { + Vertex v = vertices.next(); + if(v.id().toString() != "16752") continue; + + assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent()); + assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent()); + assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE"); + } + + Iterator<Edge> edges = tg.edges(); + while(edges.hasNext()) { + Edge e = edges.next(); + assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent()); + assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE"); + assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent()); + } + } + + @Test + public void processEdge_PropagateSetTo_ONE_TO_TWO() throws IOException { + Map<String, RelationshipCacheGenerator.TypeInfo> typeCache = new HashMap<String, RelationshipCacheGenerator.TypeInfo>() {{ + put("__Process.inputs", new RelationshipCacheGenerator.TypeInfo("dataset_process_inputs", AtlasRelationshipDef.PropagateTags.TWO_TO_ONE)); + }}; + + ElementProcessors elementProcessors = new ElementProcessors(typeCache, getPostProcessMap()); + TinkerGraph tg = TinkerGraph.open(); + GraphSONUtility gu = new GraphSONUtility(elementProcessors); + + addVertexToGraph(tg, gu, getDBV(), getTableV(), getProcessV()); + addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeProcess()); + + Iterator<Edge> edges = tg.edges(); + while(edges.hasNext()) { + Edge e = edges.next(); + assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent()); + assertEquals(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).value().toString(), "TWO_TO_ONE"); + assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE"); + assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent()); + } + } + + private Map<String, Map<String, List<String>>> getPostProcessMap() { + Map<String, Map<String, List<String>>> map = new HashMap<>(); + map.put(HIVE_TABLE_TYPE, new HashMap<>()); + + map.get(HIVE_TABLE_TYPE).put("ARRAY", new ArrayList<>()); + map.get(HIVE_TABLE_TYPE).put("MAP", new ArrayList<>()); + + map.get(HIVE_TABLE_TYPE).get("ARRAY").add(HIVE_TABLE_COLUMNS_RELATIONSHIP); + map.get(HIVE_TABLE_TYPE).get("MAP").add(HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP); + + return map; + } + + private Map<String, Map<String, List<String>>> getPostProcessMapPrimitive() { + Map<String, Map<String, List<String>>> map = new HashMap<>(); + map.put(HIVE_TABLE_TYPE, new HashMap<>()); + map.get(HIVE_TABLE_TYPE).put("MAP_PRIMITIVE", new ArrayList<>()); + map.get(HIVE_TABLE_TYPE).get("MAP_PRIMITIVE").add(HIVE_TABLE_COLUMNS_PARAMETERS_MAP); + + return map; + } + + private Map<String, Map<String, List<String>>> getNonPrimitiveArray() { + Map<String, Map<String, List<String>>> map = new HashMap<>(); + map.put(HIVE_TABLE_TYPE, new HashMap<>()); + map.get(HIVE_TABLE_TYPE).put("ARRAY", new ArrayList<>()); + map.get(HIVE_TABLE_TYPE).get("ARRAY").add(HIVE_TABLE_COLUMNS_RELATIONSHIP); + + return map; + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java index 170b1af..83e0bbe 100644 --- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java @@ -36,8 +36,9 @@ import static org.testng.AssertJUnit.assertNotNull; public class JsonNodeParsersTest extends BaseUtils { - @Test(dataProvider = "col1") - public void parseVertex(JsonNode nd) { + @Test + public void parseVertex() { + JsonNode nd = getCol1(); final int COL1_ORIGINAL_ID = 98336; Object nodeId = getId(nd); @@ -68,8 +69,9 @@ public class JsonNodeParsersTest extends BaseUtils { assertProperties(vUsingPe); } - @Test(dataProvider = "edge") - public void parseEdge(JsonNode nd) throws IOException { + @Test + public void parseEdge() { + JsonNode nd = getEdge(); final String EDGE_ORIGINAL_ID = "8k5i-35tc-acyd-1eko"; Object nodeId = getId(nd); @@ -78,12 +80,12 @@ public class JsonNodeParsersTest extends BaseUtils { JsonNodeParsers.ParseElement peVertex = new JsonNodeParsers.ParseVertex(); peVertex.setContext(graphSONUtility); - peVertex.parse(tg, cache, (JsonNode) (getDBV(null)[0][0])); - peVertex.parse(tg, cache, (JsonNode) (getTableV(null)[0][0])); + peVertex.parse(tg, cache, getDBV()); + peVertex.parse(tg, cache, getTableV()); JsonNodeParsers.ParseElement pe = new JsonNodeParsers.ParseEdge(); pe.setContext(graphSONUtility); - pe.parse(tg, cache, (JsonNode) getEdge(null)[0][0]); + pe.parse(tg, cache, getEdge()); updateParseElement(tg, pe, nodeId); http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java index cac09d2..459efd2 100644 --- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java @@ -31,8 +31,9 @@ import static org.testng.Assert.*; public class MappedElementCacheTest extends BaseUtils { - @Test(dataProvider = "col1") - public void vertexFetch(JsonNode node) { + @Test + public void vertexFetch() { + JsonNode node = getCol1(); MappedElementCache cache = new MappedElementCache(); TinkerGraph tg = TinkerGraph.open(); @@ -41,32 +42,15 @@ public class MappedElementCacheTest extends BaseUtils { Vertex vx = cache.getMappedVertex(tg, 98336); assertNotNull(vx); assertEquals(cache.lruVertexCache.size(), 1); - assertEquals(cache.lruEdgeCache.size(), 0); } @Test - public void edgeFetch() throws IOException { + public void edgeFetch() { MappedElementCache cache = new MappedElementCache(); TinkerGraph tg = TinkerGraph.open(); addEdge(tg, cache); - assertEquals(cache.lruVertexCache.size(), 2); - assertEquals(cache.lruEdgeCache.size(), 0); - } - - - @Test - public void nonExistentVertexReturnsNull() { - TinkerGraph tg = TinkerGraph.open(); - MappedElementCache cache = new MappedElementCache(); - - assertNull(cache.fetchVertex(tg, 1111)); - assertNull(cache.fetchEdge(tg, "abcd")); - } - - @DataProvider(name = "col1") - public Object[][] getCol1(ITestContext context) throws IOException { - return getJsonNodeFromFile("col-legacy.json"); + assertEquals(cache.lruVertexCache.size(), 4); } } http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java new file mode 100644 index 0000000..6227f66 --- /dev/null +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.graphdb.janus.migration; + +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.janus.migration.postProcess.PostProcessListProperty; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; +import org.eclipse.jetty.util.BlockingArrayQueue; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.BlockingQueue; + +import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +public class PostProcessListPropertyTest extends BaseUtils { + final String HIVE_TABLE_TYPE = "hive_table"; + final String HIVE_COLUMNS_PROPERTY = "hive_table.columns"; + + final String col1EdgeId = "816u-35tc-ao0l-47so"; + final String col2EdgeId = "82rq-35tc-ao0l-2glc"; + + @Test + public void noRefNoUpdate() throws IOException { + TestSetup ts = new TestSetup(); + + ts.getPostProcessListProperty().process(ts.getTable(), HIVE_TABLE_TYPE, HIVE_COLUMNS_PROPERTY); + ts.assertIncomplete(); + } + + @Test + public void refFoundVertexUpdated() throws IOException { + TestSetup ts = new TestSetup(); + assertNotNull(ts.getTable()); + + ts.getPostProcessListProperty().process(ts.getTable(), HIVE_TABLE_TYPE, HIVE_COLUMNS_PROPERTY); + ts.assertComplete(); + } + + @Test + public void updateUsingPostProcessConsumer() throws IOException { + TestSetup ts = new TestSetup(); + + BlockingQueue<Object> bc = new BlockingArrayQueue<>(); + PostProcessManager.Consumer consumer = new PostProcessManager.Consumer(bc, ts.getGraph(), + getTypePropertyMap("hive_table", HIVE_COLUMNS_PROPERTY, "ARRAY"), 5); + + Vertex tableV = fetchTableVertex(ts.getGraph()); + consumer.processItem(tableV.id()); + ts.assertComplete(); + } + + private class TestSetup { + private PostProcessListProperty postProcessListProperty; + private TinkerGraph tg; + private MappedElementCache cache; + private Vertex tableV; + + public PostProcessListProperty getPostProcessListProperty() { + return postProcessListProperty; + } + + public TinkerGraph getGraph() { + return tg; + } + + public MappedElementCache getCache() { + return cache; + } + + public Vertex getTable() { + return tableV; + } + + public TestSetup() throws IOException { + postProcessListProperty = new PostProcessListProperty(); + tg = TinkerGraph.open(); + cache = new MappedElementCache(); + addEdge(tg, cache); + tableV = fetchTableVertex(tg); + + assertSetup(); + } + + public void assertSetup() { + assertTrue(tableV.property(HIVE_COLUMNS_PROPERTY).isPresent()); + List list = (List) tableV.property(HIVE_COLUMNS_PROPERTY).value(); + + assertEquals(list.size(), 2); + assertEquals(list.get(0), col1EdgeId); + assertEquals(list.get(1), col2EdgeId); + } + + private void assertIncomplete() { + assertPropertyRemoved(HIVE_COLUMNS_PROPERTY, tableV); + + Iterator<Edge> edges = tableV.edges(Direction.OUT, getEdgeLabel(HIVE_COLUMNS_PROPERTY)); + while (edges.hasNext()) { + Edge e = edges.next(); + + assertFalse(e.property(ATTRIBUTE_INDEX_PROPERTY_KEY).isPresent()); + } + } + + private void assertComplete() { + assertPropertyRemoved(HIVE_COLUMNS_PROPERTY, tableV); + } + + private void assertPropertyRemoved(String property, Vertex tableV) { + assertFalse(tableV.property(property).isPresent()); + } + + public String getEdgeLabel(String property ) { + return Constants.INTERNAL_PROPERTY_KEY_PREFIX + property; + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/col-2-legacy.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/col-2-legacy.json b/graphdb/janus/src/test/resources/col-2-legacy.json new file mode 100644 index 0000000..f755462 --- /dev/null +++ b/graphdb/janus/src/test/resources/col-2-legacy.json @@ -0,0 +1,73 @@ +{ + "Asset.name": { + "type": "string", + "value": "col2" + }, + "hive_column.type": { + "type": "string", + "value": "string" + }, + "__modifiedBy": { + "type": "string", + "value": "anonymous" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "entityText": { + "type": "string", + "value": "hive_column owner anonymous qualifiedName stocks.test_table_view.col2@cl1 name col2 position 1 type string table " + }, + "Referenceable.qualifiedName": { + "type": "string", + "value": "stocks.test_table_view.col2@cl1" + }, + "__guid": { + "type": "string", + "value": "9cef2494-766c-4671-96a8-828dce677e7e" + }, + "__version": { + "type": "integer", + "value": 0 + }, + "__superTypeNames": { + "type": "list", + "value": [{ + "type": "string", + "value": "Asset" + }, { + "type": "string", + "value": "DataSet" + }, { + "type": "string", + "value": "Referenceable" + }] + }, + "__createdBy": { + "type": "string", + "value": "anonymous" + }, + "__typeName": { + "type": "string", + "value": "hive_column" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522693838471 + }, + "Asset.owner": { + "type": "string", + "value": "anonymous" + }, + "hive_column.position": { + "type": "integer", + "value": 1 + }, + "__timestamp": { + "type": "long", + "value": 1522693835017 + }, + "_id": 114816, + "_type": "vertex" +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/col-3-legacy.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/col-3-legacy.json b/graphdb/janus/src/test/resources/col-3-legacy.json new file mode 100644 index 0000000..b6e8c20 --- /dev/null +++ b/graphdb/janus/src/test/resources/col-3-legacy.json @@ -0,0 +1,86 @@ +{ + "Asset.name": { + "type": "string", + "value": "open" + }, + "hive_column.type": { + "type": "string", + "value": "string" + }, + "__modifiedBy": { + "type": "string", + "value": "anonymous" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "entityText": { + "type": "string", + "value": "hive_column owner anonymous qualifiedName stocks.stocks_daily.open@cl1 name open position 1 type string table Tag1 " + }, + "Referenceable.qualifiedName": { + "type": "string", + "value": "stocks.stocks_daily.open@cl1" + }, + "__traitNames": { + "type": "list", + "value": [ + { + "type": "string", + "value": "Tag1" + } + ] + }, + "__guid": { + "type": "string", + "value": "8231a95f-d062-4685-81aa-0b62401bc796" + }, + "__version": { + "type": "integer", + "value": 0 + }, + "__superTypeNames": { + "type": "list", + "value": [ + { + "type": "string", + "value": "Asset" + }, + { + "type": "string", + "value": "DataSet" + }, + { + "type": "string", + "value": "Referenceable" + } + ] + }, + "__createdBy": { + "type": "string", + "value": "anonymous" + }, + "__typeName": { + "type": "string", + "value": "hive_column" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522693815850 + }, + "Asset.owner": { + "type": "string", + "value": "anonymous" + }, + "hive_column.position": { + "type": "integer", + "value": 1 + }, + "__timestamp": { + "type": "long", + "value": 1522693815850 + }, + "_id": 16752, + "_type": "vertex" +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/edge-legacy-col.json b/graphdb/janus/src/test/resources/edge-legacy-col.json new file mode 100644 index 0000000..1e82f04 --- /dev/null +++ b/graphdb/janus/src/test/resources/edge-legacy-col.json @@ -0,0 +1,27 @@ +{ + "__modifiedBy": { + "type": "string", + "value": "anonymous" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "__createdBy": { + "type": "string", + "value": "anonymous" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522693835017 + }, + "__timestamp": { + "type": "long", + "value": 1522693835017 + }, + "_id": "816u-35tc-ao0l-47so", + "_type": "edge", + "_outV": 147504, + "_inV": 98336, + "_label": "__hive_table.columns" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col2.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/edge-legacy-col2.json b/graphdb/janus/src/test/resources/edge-legacy-col2.json new file mode 100644 index 0000000..75e0ac4 --- /dev/null +++ b/graphdb/janus/src/test/resources/edge-legacy-col2.json @@ -0,0 +1,27 @@ +{ + "__modifiedBy": { + "type": "string", + "value": "anonymous" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "__createdBy": { + "type": "string", + "value": "anonymous" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522693835017 + }, + "__timestamp": { + "type": "long", + "value": 1522693835017 + }, + "_id": "82rq-35tc-ao0l-2glc", + "_type": "edge", + "_outV": 147504, + "_inV": 114816, + "_label": "__hive_table.columns" +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col3.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/edge-legacy-col3.json b/graphdb/janus/src/test/resources/edge-legacy-col3.json new file mode 100644 index 0000000..6145ce1 --- /dev/null +++ b/graphdb/janus/src/test/resources/edge-legacy-col3.json @@ -0,0 +1,27 @@ +{ + "__modifiedBy": { + "type": "string", + "value": "anonymous" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "__createdBy": { + "type": "string", + "value": "anonymous" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522693835017 + }, + "__timestamp": { + "type": "long", + "value": 1522693835017 + }, + "_id": "816u-35tc-ao0l-47aa", + "_type": "edge", + "_outV": 147504, + "_inV": 98336, + "_label": "__hive_table.columnsMap.col3" +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col4.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/edge-legacy-col4.json b/graphdb/janus/src/test/resources/edge-legacy-col4.json new file mode 100644 index 0000000..15cdda7 --- /dev/null +++ b/graphdb/janus/src/test/resources/edge-legacy-col4.json @@ -0,0 +1,27 @@ +{ + "__modifiedBy": { + "type": "string", + "value": "anonymous" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "__createdBy": { + "type": "string", + "value": "anonymous" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522693835017 + }, + "__timestamp": { + "type": "long", + "value": 1522693835017 + }, + "_id": "82rq-35tc-ao0l-2gaa", + "_type": "edge", + "_outV": 147504, + "_inV": 114816, + "_label": "__hive_table.columnsMap.col4" +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-process.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/edge-legacy-process.json b/graphdb/janus/src/test/resources/edge-legacy-process.json new file mode 100644 index 0000000..9285958 --- /dev/null +++ b/graphdb/janus/src/test/resources/edge-legacy-process.json @@ -0,0 +1,27 @@ +{ + "__modifiedBy": { + "type": "string", + "value": "anonymous" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "__createdBy": { + "type": "string", + "value": "anonymous" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522693837285 + }, + "__timestamp": { + "type": "long", + "value": 1522693837285 + }, + "_id": "6jgh-23uw-2uqd-1elc", + "_type": "edge", + "_outV": 98312, + "_inV": 147504, + "_label": "__Process.inputs" +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-tag.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/edge-legacy-tag.json b/graphdb/janus/src/test/resources/edge-legacy-tag.json new file mode 100644 index 0000000..1bc200a --- /dev/null +++ b/graphdb/janus/src/test/resources/edge-legacy-tag.json @@ -0,0 +1,27 @@ +{ + "__modifiedBy": { + "type": "string", + "value": "admin" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "__createdBy": { + "type": "string", + "value": "admin" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522694252176 + }, + "__timestamp": { + "type": "long", + "value": 1522694252176 + }, + "_id": "2pk3la-cxc-m61h-2pk0og", + "_type": "edge", + "_outV": 16752, + "_inV": 163856752, + "_label": "Tag1" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/lineage-v-98312.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/lineage-v-98312.json b/graphdb/janus/src/test/resources/lineage-v-98312.json new file mode 100644 index 0000000..abdf9cb --- /dev/null +++ b/graphdb/janus/src/test/resources/lineage-v-98312.json @@ -0,0 +1,79 @@ +{ + "hive_column_lineage.depenendencyType": { + "type": "string", + "value": "SIMPLE" + }, + "Asset.name": { + "type": "string", + "value": "create view test_table_view partitioned on (col4) as select col1, col2, col4 from test_table:col1" + }, + "Process.inputs": { + "type": "list", + "value": [{ + "type": "string", + "value": "6jgh-23uw-2uqd-1elc" + }] + }, + "Process.outputs": { + "type": "list", + "value": [{ + "type": "string", + "value": "6ept-23uw-2wb9-47so" + }] + }, + "__modifiedBy": { + "type": "string", + "value": "anonymous" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "entityText": { + "type": "string", + "value": "hive_column_lineage outputs qualifiedName stocks.test_table_view@cl1:1522693834000:col1 inputs query name create view test_table_view partitioned on (col4) as select col1, col2, col4 from test_table:col1 depenendencyType SIMPLE " + }, + "Referenceable.qualifiedName": { + "type": "string", + "value": "stocks.test_table_view@cl1:1522693834000:col1" + }, + "__guid": { + "type": "string", + "value": "9336b7a4-9cc0-4ef0-8dc1-01fce2def6a5" + }, + "__version": { + "type": "integer", + "value": 0 + }, + "__superTypeNames": { + "type": "list", + "value": [{ + "type": "string", + "value": "Asset" + }, { + "type": "string", + "value": "Process" + }, { + "type": "string", + "value": "Referenceable" + }] + }, + "__createdBy": { + "type": "string", + "value": "anonymous" + }, + "__typeName": { + "type": "string", + "value": "hive_column_lineage" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522693837285 + }, + "__timestamp": { + "type": "long", + "value": 1522693837285 + }, + "_id": 98312, + "_type": "vertex" +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/table-v-147504.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/table-v-147504.json b/graphdb/janus/src/test/resources/table-v-147504.json index 898dce5..dc5d788 100644 --- a/graphdb/janus/src/test/resources/table-v-147504.json +++ b/graphdb/janus/src/test/resources/table-v-147504.json @@ -33,10 +33,12 @@ }, "hive_table.parameters": { "type": "list", - "value": [{ - "type": "string", - "value": "transient_lastDdlTime" - }] + "value": [ + { + "type": "string", + "value": "transient_lastDdlTime" + } + ] }, "hive_table.retention": { "type": "integer", @@ -44,10 +46,12 @@ }, "hive_table.partitionKeys": { "type": "list", - "value": [{ - "type": "string", - "value": "8dty-35tc-amfp-23xs" - }] + "value": [ + { + "type": "string", + "value": "8dty-35tc-amfp-23xs" + } + ] }, "__guid": { "type": "string", @@ -63,16 +67,20 @@ }, "__superTypeNames": { "type": "list", - "value": [{ - "type": "string", - "value": "Asset" - }, { - "type": "string", - "value": "DataSet" - }, { - "type": "string", - "value": "Referenceable" - }] + "value": [ + { + "type": "string", + "value": "Asset" + }, + { + "type": "string", + "value": "DataSet" + }, + { + "type": "string", + "value": "Referenceable" + } + ] }, "hive_table.viewExpandedText": { "type": "string", @@ -104,13 +112,37 @@ }, "hive_table.columns": { "type": "list", - "value": [{ - "type": "string", - "value": "816u-35tc-ao0l-47so" - }, { - "type": "string", - "value": "82rq-35tc-ao0l-2glc" - }] + "value": [ + { + "type": "string", + "value": "816u-35tc-ao0l-47so" + }, + { + "type": "string", + "value": "82rq-35tc-ao0l-2glc" + } + ] + }, + "hive_table.columnsMap.col3": { + "type": "string", + "value": "816u-35tc-ao0l-47aa" + }, + "hive_table.columnsMap.col4": { + "type": "string", + "value": "82rq-35tc-ao0l-2gaa" + }, + "hive_table.columnsMap": { + "type": "list", + "value": [ + { + "type": "string", + "value": "col3" + }, + { + "type": "string", + "value": "col4" + } + ] }, "__timestamp": { "type": "long", @@ -119,3 +151,4 @@ "_id": 147504, "_type": "vertex" } + http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/tag-163856752.json ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/tag-163856752.json b/graphdb/janus/src/test/resources/tag-163856752.json new file mode 100644 index 0000000..7049819 --- /dev/null +++ b/graphdb/janus/src/test/resources/tag-163856752.json @@ -0,0 +1,35 @@ +{ + "__superTypeNames": { + "type": "list", + "value": [{ + "type": "string", + "value": "[]" + }] + }, + "__modifiedBy": { + "type": "string", + "value": "admin" + }, + "__state": { + "type": "string", + "value": "ACTIVE" + }, + "__createdBy": { + "type": "string", + "value": "admin" + }, + "__typeName": { + "type": "string", + "value": "Tag1" + }, + "__modificationTimestamp": { + "type": "long", + "value": 1522694252176 + }, + "__timestamp": { + "type": "long", + "value": 1522694252176 + }, + "_id": 163856752, + "_type": "vertex" +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java index b05754f..025f89a 100644 --- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java +++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java @@ -21,16 +21,12 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.SearchFilter; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; -import org.apache.atlas.model.typedef.AtlasClassificationDef.AtlasClassificationDefs; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; -import java.io.InputStream; -import java.util.List; -import java.util.Map; /** * Interface to persistence store of TypeDef @@ -107,6 +103,4 @@ public interface AtlasTypeDefStore { AtlasBaseTypeDef getByName(String name) throws AtlasBaseException; AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException; - - void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException; } http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index 886ce77..4e9a651 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -18,7 +18,6 @@ package org.apache.atlas; -import com.sun.org.apache.xpath.internal.operations.Bool; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java index 22cd552..081376f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java @@ -18,11 +18,11 @@ package org.apache.atlas.repository.migration; -import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.impexp.AtlasImportResult; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.impexp.ImportTypeDefProcessor; import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.store.AtlasTypeDefStore; @@ -56,11 +56,11 @@ public class DataMigrationService implements Service { private final Thread thread; @Inject - public DataMigrationService(AtlasTypeDefStore typeDefStore, Configuration configuration, + public DataMigrationService(AtlasGraph graph, AtlasTypeDefStore typeDefStore, Configuration configuration, GraphBackedSearchIndexer indexer, AtlasTypeDefStoreInitializer storeInitializer, AtlasTypeRegistry typeRegistry) { this.configuration = configuration; - this.thread = new Thread(new FileImporter(typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer)); + this.thread = new Thread(new FileImporter(graph, typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer)); } @Override @@ -83,15 +83,17 @@ public class DataMigrationService implements Service { } public static class FileImporter implements Runnable { + private final AtlasGraph graph; private final AtlasTypeDefStore typeDefStore; private final String importDirectory; private final GraphBackedSearchIndexer indexer; private final AtlasTypeRegistry typeRegistry; private final AtlasTypeDefStoreInitializer storeInitializer; - public FileImporter(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, + public FileImporter(AtlasGraph graph, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, AtlasTypeDefStoreInitializer storeInitializer, String directoryName, GraphBackedSearchIndexer indexer) { + this.graph = graph; this.typeDefStore = typeDefStore; this.typeRegistry = typeRegistry; this.storeInitializer = storeInitializer; @@ -99,7 +101,16 @@ public class DataMigrationService implements Service { this.indexer = indexer; } - public void performImport() throws AtlasBaseException { + @Override + public void run() { + try { + performImport(); + } catch (AtlasBaseException e) { + LOG.error("Data Migration:", e); + } + } + + private void performImport() throws AtlasBaseException { try { if(!performAccessChecks(importDirectory)) { return; @@ -109,7 +120,7 @@ public class DataMigrationService implements Service { FileInputStream fs = new FileInputStream(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_DATA_NAME)); - typeDefStore.loadLegacyData(RelationshipCacheGenerator.get(typeRegistry), fs); + graph.importLegacyGraphSON(typeRegistry, fs); } catch (Exception ex) { LOG.error("Import failed!", ex); throw new AtlasBaseException(ex); @@ -117,11 +128,13 @@ public class DataMigrationService implements Service { } private boolean performAccessChecks(String path) { - boolean ret = false; + final boolean ret; + if(StringUtils.isEmpty(path)) { ret = false; } else { File f = new File(path); + ret = f.exists() && f.isDirectory() && f.canRead(); } @@ -137,17 +150,19 @@ public class DataMigrationService implements Service { private void performInit() throws AtlasBaseException, AtlasException { indexer.instanceIsActive(); storeInitializer.instanceIsActive(); + processIncomingTypesDef(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_TYPESDEF_NAME)); } - @VisibleForTesting - void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException { + private void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException { try { - AtlasImportResult result = new AtlasImportResult(); - String jsonStr = FileUtils.readFileToString(typesDefFile); - AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class); + AtlasImportResult result = new AtlasImportResult(); + String jsonStr = FileUtils.readFileToString(typesDefFile); + AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class); ImportTypeDefProcessor processor = new ImportTypeDefProcessor(typeDefStore, typeRegistry); + processor.processTypes(typesDef, result); + LOG.info(" types migrated: {}", result.getMetrics()); } catch (IOException e) { LOG.error("processIncomingTypesDef: Could not process file: {}! Imported data may not be usable.", typesDefFile.getName()); @@ -157,14 +172,5 @@ public class DataMigrationService implements Service { private File getFileFromImportDirectory(String importDirectory, String fileName) { return Paths.get(importDirectory, fileName).toFile(); } - - @Override - public void run() { - try { - performImport(); - } catch (AtlasBaseException e) { - LOG.error("Data Migration:", e); - } - } } } http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java b/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java deleted file mode 100644 index 2550daf..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.migration; - -import org.apache.atlas.model.typedef.AtlasRelationshipDef; -import org.apache.atlas.model.typedef.AtlasRelationshipEndDef; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasRelationshipType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.commons.lang.StringUtils; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -public class RelationshipCacheGenerator { - - public static Map<String, String> get(AtlasTypeRegistry typeRegistry) { - Map<String, String> ret = new HashMap<>(); - Collection<AtlasRelationshipType> relationshipTypes = typeRegistry.getAllRelationshipTypes(); - - for (AtlasRelationshipType rt : relationshipTypes) { - AtlasRelationshipDef rd = rt.getRelationshipDef(); - String relTypeName = rt.getTypeName(); - - add(ret, getKey(rd.getEndDef1(), rt.getEnd1Type()), relTypeName); - add(ret, getKey(rd.getEndDef2(), rt.getEnd2Type()), relTypeName); - } - - return ret; - } - - private static String getKey(AtlasRelationshipEndDef ed, AtlasEntityType rt) { - return getKey(ed.getIsLegacyAttribute(), rt.getTypeName(), ed.getName()); - } - - private static String getKey(String lhs, String rhs) { - return String.format("%s%s.%s", Constants.INTERNAL_PROPERTY_KEY_PREFIX, lhs, rhs); - } - - private static String getKey(boolean isLegacy, String typeName, String name) { - if(!isLegacy) { - return ""; - } - - return getKey(typeName, name); - } - - private static void add(Map<String, String> map, String key, String value) { - if(StringUtils.isEmpty(key) || map.containsKey(key)) { - return; - } - - map.put(key, value); - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java index 3c84e3c..67631bf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java @@ -24,7 +24,6 @@ import static org.apache.atlas.repository.Constants.TYPE_CATEGORY_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.VERTEX_TYPE; -import java.io.InputStream; import java.util.Date; import java.util.HashSet; import java.util.Iterator; @@ -49,7 +48,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.inject.Singleton; @@ -110,11 +108,6 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore { LOG.info("<== AtlasTypeDefGraphStoreV1.init()"); } - @Override - public void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException { - getAtlasGraph().loadLegacyGraphSON(relationshipCache, fs); - } - AtlasGraph getAtlasGraph() { return atlasGraph; } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java new file mode 100644 index 0000000..9451fa2 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.migration; + +import com.google.inject.Inject; +import org.apache.atlas.TestModules; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.io.IOException; + +@Guice(modules = TestModules.TestOnlyModule.class) +public class ComplexAttributesTest extends MigrationBaseAsserts { + + @Inject + public ComplexAttributesTest(AtlasGraph graph) { + super(graph); + } + + @Test + public void verify() throws IOException, AtlasBaseException { + String STRUCT_TYPE = "struct_type"; + String ENTITY_TYPE = "entity_type"; + String ENTITY_WITH_COMPLEX_COLL_TYPE = "entity_with_complex_collection_attr"; + + final int EXPECTED_TOTAL_COUNT = 214; + final int EXPECTED_ENTITY_TYPE_COUNT = 16; + final int EXPECTED_STRUCT_TYPE_COUNT = 3; + final int EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT = 1; + + runFileImporter("complex-attr_db"); + + assertTypeCountNameGuid(STRUCT_TYPE, EXPECTED_STRUCT_TYPE_COUNT,"", ""); + assertTypeCountNameGuid(ENTITY_TYPE, EXPECTED_ENTITY_TYPE_COUNT, "", ""); + assertTypeCountNameGuid(ENTITY_WITH_COMPLEX_COLL_TYPE, EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT, "", ""); + + assertEdgesWithLabel(getVertex(ENTITY_WITH_COMPLEX_COLL_TYPE, "").getEdges(AtlasEdgeDirection.OUT).iterator(),1, "__entity_with_complex_collection_attr.listOfEntities"); + assertEdgesWithLabel(getVertex(ENTITY_WITH_COMPLEX_COLL_TYPE, "").getEdges(AtlasEdgeDirection.OUT).iterator(),9, "__entity_with_complex_collection_attr.mapOfStructs"); + + assertMigrationStatus(EXPECTED_TOTAL_COUNT); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java index ac0b79d..7b23e59 100644 --- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java @@ -21,17 +21,13 @@ package org.apache.atlas.repository.migration; import com.google.inject.Inject; import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasVertex; import org.testng.annotations.Guice; import org.testng.annotations.Test; import java.io.IOException; -import java.util.List; -import static org.testng.Assert.assertEquals; @Guice(modules = TestModules.TestOnlyModule.class) public class HiveParititionTest extends MigrationBaseAsserts { @@ -41,7 +37,7 @@ public class HiveParititionTest extends MigrationBaseAsserts { super(graph); } - @Test(enabled = false) + @Test public void fileImporterTest() throws IOException, AtlasBaseException { final int EXPECTED_TOTAL_COUNT = 141; final int EXPECTED_DB_COUNT = 1; @@ -50,8 +46,6 @@ public class HiveParititionTest extends MigrationBaseAsserts { runFileImporter("parts_db"); - assertPartitionKeyProperty(getVertex("hive_table", "t1"), 1); - assertPartitionKeyProperty(getVertex("hive_table", "tv1"), 1); assertHiveVertices(EXPECTED_DB_COUNT, EXPECTED_TABLE_COUNT, EXPECTED_COLUMN_COUNT); assertTypeCountNameGuid("hive_db", 1, "parts_db", "ae30d78b-51b4-42ab-9436-8d60c8f68b95"); @@ -62,9 +56,4 @@ public class HiveParititionTest extends MigrationBaseAsserts { assertMigrationStatus(EXPECTED_TOTAL_COUNT); } - - private void assertPartitionKeyProperty(AtlasVertex vertex, int expectedCount) { - List<String> keys = GraphHelper.getListProperty(vertex, "hive_table.partitionKeys"); - assertEquals(keys.size(), expectedCount); - } } http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java index ffbf320..721bb31 100644 --- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java @@ -35,7 +35,7 @@ public class HiveStocksTest extends MigrationBaseAsserts { super(graph); } - @Test(enabled = false) + @Test public void migrateStocks() throws AtlasBaseException, IOException { final int EXPECTED_TOTAL_COUNT = 188; final int EXPECTED_DB_COUNT = 1; @@ -47,7 +47,6 @@ public class HiveStocksTest extends MigrationBaseAsserts { assertHiveVertices(EXPECTED_DB_COUNT, EXPECTED_TABLE_COUNT, EXPECTED_COLUMN_COUNT); assertTypeCountNameGuid("hive_db", 1, "stocks", "4e13b36b-9c54-4616-9001-1058221165d0"); assertTypeCountNameGuid("hive_table", 1, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774"); - assertTypeAttribute("hive_table", 7, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774", "hive_table.columns"); assertTypeCountNameGuid("hive_column", 1, "high", "d72ce4fb-6f17-4e68-aa85-967366c9e891"); assertTypeCountNameGuid("hive_column", 1, "open", "788ba8fe-b7d8-41ba-84ef-c929732924ec"); assertTypeCountNameGuid("hive_column", 1, "dt", "643a0a71-0d97-477d-a43b-7ca433f85160"); http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java index ec6e64a..4b02b7f 100644 --- a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java +++ b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java @@ -34,7 +34,6 @@ import org.testng.annotations.AfterClass; import java.io.IOException; import java.util.Iterator; -import java.util.List; import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr; import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; @@ -48,10 +47,10 @@ public class MigrationBaseAsserts { private final String R_GUID_PROPERTY_NAME = "_r__guid"; @Inject - private AtlasTypeDefStore typeDefStore; + protected AtlasTypeDefStore typeDefStore; @Inject - private AtlasTypeRegistry typeRegistry; + protected AtlasTypeRegistry typeRegistry; @Inject private AtlasTypeDefStoreInitializer storeInitializer; @@ -74,7 +73,7 @@ public class MigrationBaseAsserts { } } - private void loadTypesFromJson() throws IOException, AtlasBaseException { + protected void loadTypesFromJson() throws IOException, AtlasBaseException { loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry); loadModelFromJson("1000-Hadoop/1020-fs_model.json", typeDefStore, typeRegistry); loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry); @@ -83,7 +82,7 @@ public class MigrationBaseAsserts { protected void runFileImporter(String directoryToImport) throws IOException, AtlasBaseException { loadTypesFromJson(); String directoryName = TestResourceFileUtils.getDirectory(directoryToImport); - DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(typeDefStore, typeRegistry, + DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(graph, typeDefStore, typeRegistry, storeInitializer, directoryName, indexer); fi.run(); @@ -152,14 +151,19 @@ public class MigrationBaseAsserts { assertEquals(count, expectedItems, String.format("%s", edgeTypeName)); } - protected void assertTypeAttribute(String typeName, int expectedSize, String name, String guid, String propertyName) { - AtlasVertex v = getVertex(typeName, name); - String guidActual = GraphHelper.getGuid(v); - List list = (List) GraphHelper.getProperty(v, propertyName); + protected void assertEdgesWithLabel(Iterator<AtlasEdge> results, int startIdx, String edgeTypeName) { + int count = 0; + AtlasEdge e = null; + for (Iterator<AtlasEdge> it = results; it.hasNext() && count < startIdx; count++) { + e = it.next(); + } - assertEquals(guidActual, guid); - assertNotNull(list); - assertEquals(list.size(), expectedSize); + assertNotNull(GraphHelper.getProperty(e, R_GUID_PROPERTY_NAME)); + assertNotNull(GraphHelper.getProperty(e, "tagPropagation")); + + if(StringUtils.isNotEmpty(edgeTypeName)) { + assertEquals(e.getLabel(), edgeTypeName, edgeTypeName); + } } protected void assertTypeCountNameGuid(String typeName, int expectedItems, String name, String guid) { http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java new file mode 100644 index 0000000..362990f --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.migration; + +import com.google.inject.Inject; +import org.apache.atlas.TestModules; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.typedef.AtlasRelationshipDef; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.janus.migration.RelationshipCacheGenerator; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasRelationshipType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.v1.typesystem.types.utils.TypesUtil; +import org.apache.commons.lang.StringUtils; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO; +import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +@Guice(modules = TestModules.TestOnlyModule.class) +public class RelationshipCacheGeneratorTest { + + @Inject + private AtlasTypeDefStore typeDefStore; + + @Inject + private AtlasTypeRegistry typeRegistry; + + @BeforeClass + public void setup() throws IOException, AtlasBaseException { + loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry); + loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry); + } + + @Test + public void createLookup() { + final String PROCESS_INPUT_KEY = "__Process.inputs"; + final String PROCESS_OUTPUT_KEY = "__Process.outputs"; + String ONE_TO_TWO_STR = ONE_TO_TWO.toString(); + String TWO_TO_ONE_STR = TWO_TO_ONE.toString(); + + Map<String, RelationshipCacheGenerator.TypeInfo> cache = RelationshipCacheGenerator.get(typeRegistry); + assertEquals(cache.size(), getLegacyAttributeCount() - 1); + for (Map.Entry<String, RelationshipCacheGenerator.TypeInfo> entry : cache.entrySet()) { + assertTrue(StringUtils.isNotEmpty(entry.getKey())); + assertTrue(entry.getKey().startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX), entry.getKey()); + } + + assertEquals(cache.get(PROCESS_INPUT_KEY).getTypeName(), "dataset_process_inputs"); + assertEquals(cache.get(PROCESS_INPUT_KEY).getPropagateTags(), ONE_TO_TWO_STR); + + assertEquals(cache.get(PROCESS_OUTPUT_KEY).getTypeName(), "process_dataset_outputs"); + assertEquals(cache.get(PROCESS_OUTPUT_KEY).getPropagateTags(), TWO_TO_ONE_STR); + } + + private int getLegacyAttributeCount() { + int count = 0; + for (AtlasRelationshipType rt : typeRegistry.getAllRelationshipTypes()) { + AtlasRelationshipDef rd = rt.getRelationshipDef(); + if(rd.getEndDef1().getIsLegacyAttribute()) { + count++; + } + + if(rd.getEndDef2().getIsLegacyAttribute()) { + count++; + } + } + + return count; + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java deleted file mode 100644 index 48e5391..0000000 --- a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository.migration; - -import com.google.inject.Inject; -import org.apache.atlas.TestModules; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.typedef.AtlasRelationshipDef; -import org.apache.atlas.repository.Constants; -import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.store.AtlasTypeDefStore; -import org.apache.atlas.type.AtlasRelationshipType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.commons.lang.StringUtils; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; -import org.jcodings.util.Hash; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.util.Collection; -import java.util.Map; - -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; - -@Guice(modules = TestModules.TestOnlyModule.class) -public class RelationshipMappingTest { - - @Inject - private AtlasTypeDefStore typeDefStore; - - @Inject - private AtlasTypeRegistry typeRegistry; - - @BeforeClass - public void setup() throws IOException, AtlasBaseException { - loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry); - loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry); - } - - @Test - public void createLookup() { - Map<String, String> cache = RelationshipCacheGenerator.get(typeRegistry); - assertEquals(cache.size(), getLegacyAttributeCount() - 1); - for (Map.Entry<String, String> entry : cache.entrySet()) { - assertTrue(StringUtils.isNotEmpty(entry.getKey())); - assertTrue(entry.getKey().startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX), entry.getKey()); - } - } - - private int getLegacyAttributeCount() { - int count = 0; - for (AtlasRelationshipType rt : typeRegistry.getAllRelationshipTypes()) { - AtlasRelationshipDef rd = rt.getRelationshipDef(); - if(rd.getEndDef1().getIsLegacyAttribute()) { - count++; - } - - if(rd.getEndDef2().getIsLegacyAttribute()) { - count++; - } - } - - return count; - } -}