ATLAS-639 Exception for lineage request (svimal2106 via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/67acb9d6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/67acb9d6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/67acb9d6 Branch: refs/heads/master Commit: 67acb9d6edf3c62ddc63299bb7bb304b950b7f7d Parents: f672aae Author: Shwetha GS <[email protected]> Authored: Wed Jul 20 12:11:08 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Wed Jul 20 12:11:08 2016 +0530 ---------------------------------------------------------------------- .../org/apache/atlas/hive/hook/HiveHookIT.java | 4 +- release-log.txt | 1 + .../org/apache/atlas/query/GremlinQuery.scala | 5 +- .../org/apache/atlas/BaseRepositoryTest.java | 10 ++ .../discovery/DataSetLineageServiceTest.java | 16 +++ .../GraphBackedDiscoveryServiceTest.java | 100 +++++++++---------- 6 files changed, 82 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index a15aa49..573f7f5 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -1620,7 +1620,7 @@ public class HiveHookIT { expectedProps.put("testPropKey2", "testPropValue2"); //Add another property query = String.format(fmtQuery, entityName, SET_OP, getSerializedProps(expectedProps)); - runCommand(query); + runCommandWithDelay(query, 1000); verifyEntityProperties(entityType, entityName, expectedProps, false); if (entityType != Entity.Type.DATABASE) { @@ -1628,7 +1628,7 @@ public class HiveHookIT { //Unset all the props StringBuilder sb = new StringBuilder("'"); query = String.format(fmtQuery, entityName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb, expectedProps.keySet()).append('\'')); - runCommand(query); + runCommandWithDelay(query, 1000); verifyEntityProperties(entityType, entityName, expectedProps, true); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 40de319..ef9f67f 100644 --- a/release-log.txt +++ b/release-log.txt @@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES: ALL CHANGES: +ATLAS-639 Exception for lineage request (svimal2106 via shwethags) ATLAS-1022 Update typesystem wiki with details (yhemanth via shwethags) ATLAS-1021 Update Atlas architecture wiki (yhemanth via sumasai) ATLAS-957 Atlas is not capturing topologies that have $ in the data payload (shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala index d76b173..ee221c9 100755 --- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala +++ b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala @@ -280,9 +280,10 @@ class GremlinTranslator(expr: Expression, val inputQry = genQuery(input, inSelect) val loopingPathGExpr = genQuery(loopExpr, inSelect) val loopGExpr = s"""loop("${input.asInstanceOf[AliasExpression].alias}")""" - val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{true}" + val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{it.path.contains(it.object)?false:true}" val loopObjectGExpr = gPersistenceBehavior.loopObjectExpression(input.dataType) - s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}""" + val enablePathExpr = s".enablePath()" + s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}${enablePathExpr}""" } case BackReference(alias, _, _) => if (inSelect) gPersistenceBehavior.fieldPrefixInSelect else s"""back("$alias")""" http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java index 4d3f165..73d9fc5 100644 --- a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java @@ -224,6 +224,16 @@ public class BaseRepositoryTest { table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed", salesFactColumns, "Metric"); + Id circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric"); + + Id circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric"); + + loadProcess("circularLineage1", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable1), + ImmutableList.of(circularLineageTable2), "create table as select ", "plan", "id", "graph", "ETL"); + + loadProcess("circularLineage2", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable2), + ImmutableList.of(circularLineageTable1), "create table as select ", "plan", "id", "graph", "ETL"); + loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java index 89e9b41..460a88f 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java @@ -176,6 +176,22 @@ public class DataSetLineageServiceTest extends BaseRepositoryTest { } @Test + public void testCircularLineage() throws Exception{ + JSONObject results = new JSONObject(lineageService.getInputsGraph("table2")); + assertNotNull(results); + System.out.println("inputs graph = " + results); + + JSONObject values = results.getJSONObject("values"); + assertNotNull(values); + + final JSONObject vertices = values.getJSONObject("vertices"); + Assert.assertEquals(vertices.length(), 2); + + final JSONObject edges = values.getJSONObject("edges"); + Assert.assertEquals(edges.length(), 4); + } + + @Test public void testGetInputsGraphForEntity() throws Exception { ITypedReferenceableInstance entity = repository.getEntityDefinition(HIVE_TABLE_TYPE, "name", "sales_fact_monthly_mv"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index c27894e..313a2ae 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -316,7 +316,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db hive_db.name = \"Reporting\"", 1}, {"hive_db where hive_db.name=\"Reporting\" select name, owner", 1}, {"hive_db has name", 3}, - {"hive_db, hive_table", 8}, + {"hive_db, hive_table", 10}, {"View is JdbcAccess", 2}, {"hive_db as db1, hive_table where db1.name = \"Reporting\"", 0}, //Not working - ATLAS-145 // - Final working query -> discoveryService.searchByGremlin("L:{_var_0 = [] as Set;g.V().has(\"__typeName\", \"hive_db\").fill(_var_0);g.V().has(\"__superTypeNames\", \"hive_db\").fill(_var_0);_var_0._().as(\"db1\").in(\"__hive_table.db\").back(\"db1\").and(_().has(\"hive_db.name\", T.eq, \"Reporting\")).toList()}") @@ -324,20 +324,20 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db, hive_process has name"}, //Invalid query {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System.currentTimeMillis()} */ - {"from hive_table", 8}, - {"hive_table", 8}, + {"from hive_table", 10}, + {"hive_table", 10}, {"hive_table isa Dimension", 3}, - {"hive_column where hive_column isa PII", 6}, + {"hive_column where hive_column isa PII", 8}, {"View is Dimension" , 2}, // {"hive_column where hive_column isa PII select hive_column.name", 6}, //Not working - ATLAS-175 - {"hive_column select hive_column.name", 29}, - {"hive_column select name", 29}, - {"hive_column where hive_column.name=\"customer_id\"", 4}, - {"from hive_table select hive_table.name", 8}, + {"hive_column select hive_column.name", 37}, + {"hive_column select name",37}, + {"hive_column where hive_column.name=\"customer_id\"", 6}, + {"from hive_table select hive_table.name", 10}, {"hive_db where (name = \"Reporting\")", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1", 1}, {"hive_db where hive_db is JdbcAccess", 0}, //Not supposed to work - {"hive_db hive_table", 8}, + {"hive_db hive_table", 10}, {"hive_db where hive_db has name", 3}, {"hive_db as db1 hive_table where (db1.name = \"Reporting\")", 0}, //Not working -> ATLAS-145 {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ", 1}, @@ -359,9 +359,9 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { // trait searches {"Dimension", 5}, {"JdbcAccess", 2}, - {"ETL", 3}, - {"Metric", 5}, - {"PII", 6}, + {"ETL", 5}, + {"Metric", 9}, + {"PII", 8}, {"`Log Data`", 4}, /* Lineage queries are fired through ClosureQuery and are tested through HiveLineageJerseyResourceIt in webapp module. @@ -388,7 +388,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { return new Object[][]{ {"hive_column limit 10 ", 10}, {"hive_column select hive_column.name limit 10 ", 10}, - {"hive_column select hive_column.name withPath", 29}, + {"hive_column select hive_column.name withPath", 37}, {"hive_column select hive_column.name limit 10 withPath", 10}, {"from hive_db", 3}, {"from hive_db limit 2", 2}, @@ -405,10 +405,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db has name limit 2 offset 1", 2}, {"hive_db has name limit 10 offset 1", 2}, {"hive_db has name limit 10 offset 0", 3}, - {"hive_db, hive_table", 8}, + {"hive_db, hive_table", 10}, {"hive_db, hive_table limit 5", 5}, {"hive_db, hive_table limit 5 offset 0", 5}, - {"hive_db, hive_table limit 5 offset 5", 3}, + {"hive_db, hive_table limit 5 offset 5", 5}, {"View is JdbcAccess", 2}, {"View is JdbcAccess limit 1", 1}, @@ -416,13 +416,13 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db as db1, hive_table where db1.name = \"Reporting\"", 0}, //Not working - ATLAS-145 - {"from hive_table", 8}, + {"from hive_table", 10}, {"from hive_table limit 5", 5}, - {"from hive_table limit 5 offset 5", 3}, + {"from hive_table limit 5 offset 5", 5}, - {"hive_table", 8}, + {"hive_table", 10}, {"hive_table limit 5", 5}, - {"hive_table limit 5 offset 5", 3}, + {"hive_table limit 5 offset 5", 5}, {"hive_table isa Dimension", 3}, {"hive_table isa Dimension limit 2", 2}, @@ -430,10 +430,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_table isa Dimension limit 2 offset 1", 2}, {"hive_table isa Dimension limit 3 offset 1", 2}, - {"hive_column where hive_column isa PII", 6}, + {"hive_column where hive_column isa PII", 8}, {"hive_column where hive_column isa PII limit 5", 5}, {"hive_column where hive_column isa PII limit 5 offset 1", 5}, - {"hive_column where hive_column isa PII limit 5 offset 5", 1}, + {"hive_column where hive_column isa PII limit 5 offset 5", 3}, {"View is Dimension" , 2}, @@ -441,31 +441,31 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"View is Dimension limit 1 offset 1" , 1}, {"View is Dimension limit 10 offset 1" , 1}, - {"hive_column select hive_column.name", 29}, + {"hive_column select hive_column.name", 37}, {"hive_column select hive_column.name limit 5", 5}, - {"hive_column select hive_column.name limit 5 offset 28", 1}, + {"hive_column select hive_column.name limit 5 offset 36", 1}, - {"hive_column select name", 29}, + {"hive_column select name", 37}, {"hive_column select name limit 5", 5}, - {"hive_column select name limit 5 offset 28 ", 1}, + {"hive_column select name limit 5 offset 36 ", 1}, - {"hive_column where hive_column.name=\"customer_id\"", 4}, + {"hive_column where hive_column.name=\"customer_id\"", 6}, {"hive_column where hive_column.name=\"customer_id\" limit 2", 2}, {"hive_column where hive_column.name=\"customer_id\" limit 2 offset 1", 2}, - {"hive_column where hive_column.name=\"customer_id\" limit 10 offset 3", 1}, + {"hive_column where hive_column.name=\"customer_id\" limit 10 offset 3", 3}, - {"from hive_table select hive_table.name", 8}, + {"from hive_table select hive_table.name", 10}, {"from hive_table select hive_table.name limit 5", 5}, - {"from hive_table select hive_table.name limit 5 offset 5", 3}, + {"from hive_table select hive_table.name limit 5 offset 5", 5}, {"hive_db where (name = \"Reporting\")", 1}, {"hive_db where (name = \"Reporting\") limit 10", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1 limit 10", 1}, {"hive_db where hive_db is JdbcAccess", 0}, //Not supposed to work - {"hive_db hive_table", 8}, + {"hive_db hive_table", 10}, {"hive_db hive_table limit 5", 5}, - {"hive_db hive_table limit 5 offset 5", 3}, + {"hive_db hive_table limit 5 offset 5", 5}, {"hive_db where hive_db has name", 3}, {"hive_db where hive_db has name limit 5", 3}, {"hive_db where hive_db has name limit 2 offset 0", 2}, @@ -499,24 +499,24 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"JdbcAccess limit 2 offset 1", 1}, {"JdbcAccess limit 1", 1}, - {"ETL", 3}, + {"ETL", 5}, {"ETL limit 2", 2}, {"ETL limit 1", 1}, {"ETL limit 1 offset 0", 1}, {"ETL limit 2 offset 1", 2}, - {"Metric", 5}, - {"Metric limit 10", 5}, + {"Metric", 9}, + {"Metric limit 10", 9}, {"Metric limit 2", 2}, - {"Metric limit 10 offset 1", 4}, + {"Metric limit 10 offset 1", 8}, - {"PII", 6}, - {"PII limit 10", 6}, + {"PII", 8}, + {"PII limit 10", 8}, {"PII limit 2", 2}, - {"PII limit 10 offset 1", 5}, - + {"PII limit 10 offset 1", 7}, + {"`Log Data`", 4}, {"`Log Data` limit 3", 3}, {"`Log Data` limit 10 offset 2", 2}, @@ -553,10 +553,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_db where hive_db.name=\"Reporting\" select name, owner orderby 'hive_db.name' ", 1, "name", isAscending}, {"hive_db has name orderby 'hive_db.owner' limit 10 offset 0", 3, "owner", isAscending}, - {"from hive_table orderby 'hive_table.owner'", 8, "owner", isAscending}, + {"from hive_table orderby 'hive_table.owner'", 10, "owner", isAscending}, {"from hive_table orderby 'hive_table.owner' limit 8", 8, "owner", isAscending}, - {"hive_table orderby 'hive_table.owner'", 8, "owner", isAscending}, + {"hive_table orderby 'hive_table.owner'", 10, "owner", isAscending}, {"hive_table orderby 'hive_table.owner' limit 8", 8, "owner", isAscending}, {"hive_table orderby 'hive_table.owner' limit 8 offset 0", 8, "owner", isAscending}, {"hive_table orderby 'hive_table.owner' desc limit 8 offset 0", 8, "owner", !isAscending}, @@ -566,29 +566,29 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { {"hive_table isa Dimension orderby 'hive_table.owner' limit 3 offset 0", 3, "owner", isAscending}, {"hive_table isa Dimension orderby 'hive_table.owner' desc limit 3 offset 0", 3, "owner", !isAscending}, - {"hive_column where hive_column isa PII orderby 'hive_column.name'", 6, "name", isAscending}, + {"hive_column where hive_column isa PII orderby 'hive_column.name'", 8, "name", isAscending}, {"hive_column where hive_column isa PII orderby 'hive_column.name' limit 5", 5, "name", isAscending}, {"hive_column where hive_column isa PII orderby 'hive_column.name' limit 5 offset 1", 5, "name", isAscending}, {"hive_column where hive_column isa PII orderby 'hive_column.name' desc limit 5 offset 1", 5, "name", !isAscending}, - {"hive_column select hive_column.name orderby 'hive_column.name' ", 29, "hive_column.name", isAscending}, + {"hive_column select hive_column.name orderby 'hive_column.name' ", 37, "hive_column.name", isAscending}, {"hive_column select hive_column.name orderby 'hive_column.name' limit 5", 5, "hive_column.name", isAscending}, {"hive_column select hive_column.name orderby 'hive_column.name' desc limit 5", 5, "hive_column.name", !isAscending}, - {"hive_column select hive_column.name orderby 'hive_column.name' limit 5 offset 28", 1, "hive_column.name", isAscending}, + {"hive_column select hive_column.name orderby 'hive_column.name' limit 5 offset 36", 1, "hive_column.name", isAscending}, - {"hive_column select name orderby 'hive_column.name'", 29, "name", isAscending}, + {"hive_column select name orderby 'hive_column.name'", 37, "name", isAscending}, {"hive_column select name orderby 'hive_column.name' limit 5", 5, "name", isAscending}, - {"hive_column select name orderby 'hive_column.name' desc", 29, "name", !isAscending}, + {"hive_column select name orderby 'hive_column.name' desc", 37, "name", !isAscending}, - {"hive_column where hive_column.name=\"customer_id\" orderby 'hive_column.name'", 4, "name", isAscending}, + {"hive_column where hive_column.name=\"customer_id\" orderby 'hive_column.name'", 6, "name", isAscending}, {"hive_column where hive_column.name=\"customer_id\" orderby 'hive_column.name' limit 2", 2, "name", isAscending}, {"hive_column where hive_column.name=\"customer_id\" orderby 'hive_column.name' limit 2 offset 1", 2, "name", isAscending}, - {"from hive_table select owner orderby 'hive_table.owner'", 8, "owner", isAscending}, + {"from hive_table select owner orderby 'hive_table.owner'", 10, "owner", isAscending}, {"from hive_table select owner orderby 'hive_table.owner' limit 5", 5, "owner", isAscending}, {"from hive_table select owner orderby 'hive_table.owner' desc limit 5", 5, "owner", !isAscending}, - {"from hive_table select owner orderby 'hive_table.owner' limit 5 offset 5", 3, "owner", isAscending}, + {"from hive_table select owner orderby 'hive_table.owner' limit 5 offset 5", 5, "owner", isAscending}, {"hive_db where (name = \"Reporting\") orderby 'hive_db.name'", 1, "name", isAscending}, {"hive_db where (name = \"Reporting\") orderby 'hive_db.name' limit 10", 1, "name", isAscending}, @@ -617,7 +617,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { //Test if proeprty is not defined. it should not fail the query - {"hive_table orderby 'hive_table.owner_notdefined'", 8, null, isAscending}, + {"hive_table orderby 'hive_table.owner_notdefined'", 10, null, isAscending}, }; }
