Repository: incubator-atlas Updated Branches: refs/heads/0.8-incubating 034f29283 -> b6c8ee160
ATLAS-1807 : Enhance DSL query to support like operator for wildcard search (cherry picked from commit f2255da116af44da4c1c86c585fe7428d218691a) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/b6c8ee16 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/b6c8ee16 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/b6c8ee16 Branch: refs/heads/0.8-incubating Commit: b6c8ee1608c8d3e682f43080ef05c43ae07338d7 Parents: 034f292 Author: Sarath Subramanian <[email protected]> Authored: Tue May 16 17:33:02 2017 -0700 Committer: Sarath Subramanian <[email protected]> Committed: Tue May 16 17:35:23 2017 -0700 ---------------------------------------------------------------------- .../gremlin/Gremlin2ExpressionFactory.java | 32 ++++++++++++++++++++ .../gremlin/Gremlin3ExpressionFactory.java | 32 ++++++++++++++++++++ .../atlas/gremlin/GremlinExpressionFactory.java | 4 +++ .../org/apache/atlas/query/GremlinQuery.scala | 4 +++ .../org/apache/atlas/query/QueryParser.scala | 3 +- .../GraphBackedDiscoveryServiceTest.java | 16 ++++++++++ .../EntityDiscoveryJerseyResourceIT.java | 18 ++++++++++- 7 files changed, 107 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6c8ee16/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java index 807dd05..5f8bb80 100644 --- a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java +++ b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java @@ -40,6 +40,7 @@ import org.apache.atlas.groovy.LogicalExpression.LogicalOperator; import org.apache.atlas.groovy.RangeExpression; import org.apache.atlas.groovy.TernaryOperatorExpression; import org.apache.atlas.groovy.TraversalStepType; +import org.apache.atlas.query.Expressions; import org.apache.atlas.query.GraphPersistenceStrategies; import org.apache.atlas.query.TypeUtils.FieldInfo; import org.apache.atlas.typesystem.types.IDataType; @@ -147,6 +148,37 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory { return new FunctionCallExpression(TraversalStepType.FILTER, parent, HAS_METHOD, propertyNameExpr, op, requiredValue); } + @Override + public GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent, String propertyName, GroovyExpression propertyValue) throws AtlasException { + GroovyExpression itExpr = getItVariable(); + GroovyExpression nameExpr = new FieldExpression(itExpr, propertyName); + GroovyExpression matchesExpr = new FunctionCallExpression(nameExpr, MATCHES, escapePropertyValue(propertyValue)); + GroovyExpression closureExpr = new ClosureExpression(matchesExpr); + GroovyExpression filterExpr = new FunctionCallExpression(parent, FILTER_METHOD, closureExpr); + + return filterExpr; + } + + private GroovyExpression escapePropertyValue(GroovyExpression propertyValue) { + GroovyExpression ret = propertyValue; + + if (propertyValue instanceof LiteralExpression) { + LiteralExpression exp = (LiteralExpression) propertyValue; + + if (exp != null && exp.getValue() instanceof String) { + String stringValue = (String) exp.getValue(); + + // replace '*' with ".*", replace '?' with '.' + stringValue = stringValue.replaceAll("\\*", ".*") + .replaceAll("\\?", "."); + + ret = new LiteralExpression(stringValue); + } + } + + return ret; + } + private GroovyExpression gremlin2CompOp(String op) throws AtlasException { GroovyExpression tExpr = new IdentifierExpression("T"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6c8ee16/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java index 9f68c9a..aaec6fe 100644 --- a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java +++ b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java @@ -29,6 +29,7 @@ import org.apache.atlas.groovy.ClosureExpression; import org.apache.atlas.groovy.ComparisonExpression; import org.apache.atlas.groovy.ComparisonExpression.ComparisonOperator; import org.apache.atlas.groovy.ComparisonOperatorExpression; +import org.apache.atlas.groovy.FieldExpression; import org.apache.atlas.groovy.FunctionCallExpression; import org.apache.atlas.groovy.GroovyExpression; import org.apache.atlas.groovy.IdentifierExpression; @@ -244,6 +245,37 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory { } @Override + public GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent, String propertyName, GroovyExpression propertyValue) throws AtlasException { + GroovyExpression itExpr = getItVariable(); + GroovyExpression nameExpr = new FieldExpression(itExpr, propertyName); + GroovyExpression matchesExpr = new FunctionCallExpression(nameExpr, MATCHES, escapePropertyValue(propertyValue)); + GroovyExpression closureExpr = new ClosureExpression(matchesExpr); + GroovyExpression filterExpr = new FunctionCallExpression(parent, FILTER_METHOD, closureExpr); + + return filterExpr; + } + + private GroovyExpression escapePropertyValue(GroovyExpression propertyValue) { + GroovyExpression ret = propertyValue; + + if (propertyValue instanceof LiteralExpression) { + LiteralExpression exp = (LiteralExpression) propertyValue; + + if (exp != null && exp.getValue() instanceof String) { + String stringValue = (String) exp.getValue(); + + // replace '*' with ".*", replace '?' with '.' + stringValue = stringValue.replaceAll("\\*", ".*") + .replaceAll("\\?", "."); + + ret = new LiteralExpression(stringValue); + } + } + + return ret; + } + + @Override protected GroovyExpression initialExpression(GroovyExpression varExpr, GraphPersistenceStrategies s) { // this bit of groovy magic converts the set of vertices in varName into http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6c8ee16/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java b/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java index ff5a58c..d603150 100644 --- a/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java +++ b/repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java @@ -76,6 +76,7 @@ public abstract class GremlinExpressionFactory { protected static final String SELECT_METHOD = "select"; protected static final String ORDER_METHOD = "order"; protected static final String FILL_METHOD = "fill"; + protected static final String MATCHES = "matches"; public static final GremlinExpressionFactory INSTANCE = AtlasGraphProvider.getGraphInstance() .getSupportedGremlinVersion() == GremlinVersion.THREE ? new Gremlin3ExpressionFactory() @@ -182,6 +183,9 @@ public abstract class GremlinExpressionFactory { public abstract GroovyExpression generateHasExpression(GraphPersistenceStrategies s, GroovyExpression parent, String propertyName, String symbol, GroovyExpression requiredValue, FieldInfo fInfo) throws AtlasException; + public abstract GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent, String propertyName, + GroovyExpression propertyValue) throws AtlasException; + /** * Generates a range expression * http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6c8ee16/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala index 3a310a7..37015d8 100644 --- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala +++ b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala @@ -437,6 +437,10 @@ class GremlinTranslator(expr: Expression, genQuery(null, l, inClosure); } + if (symb == "like") { + return GremlinExpressionFactory.INSTANCE.generateLikeExpressionUsingFilter(childExpr, qualifiedPropertyName, persistentExprValue); + } + return GremlinExpressionFactory.INSTANCE.generateHasExpression(gPersistenceBehavior, childExpr, qualifiedPropertyName, c.symbol, persistentExprValue, fInfo); } case fil@FilterExpression(child, condExpr) => { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6c8ee16/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala b/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala index 8d454e9..7b7cd98 100755 --- a/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala +++ b/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala @@ -75,6 +75,7 @@ trait QueryKeywords { protected val SUM = Keyword("sum") protected val BY = Keyword("by") protected val ORDER = Keyword("order") + protected val LIKE = Keyword("like") } trait ExpressionUtils { @@ -312,7 +313,7 @@ object QueryParser extends StandardTokenParsers with QueryKeywords with Expressi def exprRight = (AND | OR) ~ compE ^^ { case op ~ c => (op, c)} def compE = - arithE ~ (LT | LTE | EQ | NEQ | GT | GTE) ~ arithE ^^ { case l ~ op ~ r => l.compareOp(op)(r)} | + arithE ~ (LT | LTE | EQ | NEQ | GT | GTE | LIKE) ~ arithE ^^ { case l ~ op ~ r => l.compareOp(op)(r)} | arithE ~ (ISA | IS) ~ ident ^^ { case l ~ i ~ t => l.isTrait(t)} | arithE ~ HAS ~ ident ^^ { case l ~ i ~ f => l.hasField(f)} | arithE | countClause | maxClause | minClause | sumClause http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6c8ee16/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index f3fdf08..be72955 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -268,6 +268,22 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest { assertEquals(entityState, Id.EntityState.ACTIVE.name()); } + @DataProvider(name = "dslLikeQueriesProvider") + private Object[][] createDslLikeQueries() { + return new Object[][]{ + {"hive_table where name like \"sa?es*\"", 3}, + {"hive_db where name like \"R*\"", 1}, + {"hive_db where hive_db.name like \"R???rt?*\" or hive_db.name like \"S?l?s\" or hive_db.name like\"Log*\"", 3}, + {"hive_db where hive_db.name like \"R???rt?*\" and hive_db.name like \"S?l?s\" and hive_db.name like\"Log*\"", 0}, + {"hive_table where name like 'sales*', db where name like 'Sa?es'", 1}, + }; + } + + @Test(dataProvider = "dslLikeQueriesProvider") + public void testDslSearchUsingLikeOperator(String dslQuery, Integer expectedNumRows) throws Exception { + runQuery(dslQuery, expectedNumRows, 50, 0); + } + @Test(expectedExceptions = Throwable.class) public void testSearchByDSLBadQuery() throws Exception { String dslQuery = "from blah"; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b6c8ee16/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java index db3a7c3..a51f371 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java @@ -54,7 +54,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT { @BeforeClass public void setUp() throws Exception { super.setUp(); - dbName = "db" + randomString(); + dbName = "database" + randomString(); createTypes(); createInstance(createHiveDBInstanceBuiltIn(dbName)); } @@ -145,6 +145,22 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT { } @Test + public void testLikeSearchUsingDSL() throws Exception { + String dslQuery = DATABASE_TYPE_BUILTIN + " where " + QUALIFIED_NAME + " like \"da?a*\""; + + AtlasSearchResult searchResult = atlasClientV2.dslSearch(dslQuery); + assertNotNull(searchResult); + + List<AtlasEntityHeader> entities = searchResult.getEntities(); + assertNotNull(entities); + assertEquals(entities.size(), 1); + + AtlasEntityHeader dbEntity = entities.get(0); + assertEquals(dbEntity.getTypeName(), DATABASE_TYPE_BUILTIN); + assertEquals(dbEntity.getDisplayText(), dbName); + } + + @Test public void testSearchFullTextOnDSLFailure() throws Exception { String query = "*"; AtlasSearchResult searchResult = atlasClientV2.fullTextSearch(query);
