This is an automated email from the ASF dual-hosted git repository. pinal pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new 37d3c6d ATLAS-4332 : DSL Query : query with like operator and / in search text throws 500 37d3c6d is described below commit 37d3c6d99c7db347280a8d9098fa5327f7dd4dd1 Author: Pinal <pinal-shah> AuthorDate: Wed Jun 9 19:36:06 2021 +0530 ATLAS-4332 : DSL Query : query with like operator and / in search text throws 500 Signed-off-by: Pinal <pinal-shah> --- .../apache/atlas/query/GremlinQueryComposer.java | 3 ++- .../org/apache/atlas/query/IdentifierHelper.java | 23 ++++++++++++++++++++++ .../test/java/org/apache/atlas/BasicTestSetup.java | 9 +++++---- .../org/apache/atlas/query/BaseDSLComposer.java | 2 ++ .../org/apache/atlas/query/DSLQueriesTest.java | 6 ++++++ .../atlas/query/GremlinQueryComposerTest.java | 3 +++ 6 files changed, 41 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java index 320acbe..cff7aff 100644 --- a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java +++ b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java @@ -218,7 +218,8 @@ public class GremlinQueryComposer { final AtlasStructDef.AtlasAttributeDef.IndexType indexType = attribute.getAttributeDef().getIndexType(); if (indexType == AtlasStructDef.AtlasAttributeDef.IndexType.STRING || !containsNumberAndLettersOnly(rhs)) { - add(GremlinClause.STRING_CONTAINS, getPropertyForClause(lhsI), IdentifierHelper.getFixedRegEx(rhs)); + String escapeRhs = IdentifierHelper.escapeCharacters(IdentifierHelper.getFixedRegEx(rhs)); + add(GremlinClause.STRING_CONTAINS, getPropertyForClause(lhsI), escapeRhs); } else { add(GremlinClause.TEXT_CONTAINS, getPropertyForClause(lhsI), IdentifierHelper.getFixedRegEx(rhs)); } diff --git a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java index c53a324..d2906ea 100644 --- a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java +++ b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java @@ -25,6 +25,9 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdg import org.apache.atlas.type.AtlasType; import org.apache.commons.lang.StringUtils; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -33,6 +36,8 @@ public class IdentifierHelper { private static final Pattern SINGLE_QUOTED_IDENTIFIER = Pattern.compile("'(\\w[\\w\\d\\.\\s]*)'"); private static final Pattern DOUBLE_QUOTED_IDENTIFIER = Pattern.compile("\"(\\w[\\w\\d\\.\\s]*)\""); private static final Pattern BACKTICK_QUOTED_IDENTIFIER = Pattern.compile("`(\\w[\\w\\d\\.\\s]*)`"); + private static final Character[] ESCAPE_CHARS = new Character[] {'+', '@', '#', '&', '|', '(', ')', '{', '}', '[', ']', '~', '\\', '/'}; + private static final Set<Character> ESCAPE_CHARACTERS_SET = new HashSet<>(Arrays.asList(ESCAPE_CHARS)); public static String get(String quotedIdentifier) { String ret; @@ -116,6 +121,24 @@ public class IdentifierHelper { return s.replace("*", ".*").replace('?', '.'); } + public static String escapeCharacters(String value) { + if (StringUtils.isEmpty(value)) { + return value; + } + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + + if (c != '*' && ESCAPE_CHARACTERS_SET.contains(c)) { + sb.append('\\'); + } + sb.append(c); + } + + return sb.toString(); + } + public static String removeWildcards(String s) { return removeQuotes(s).replace("*", "").replace("?", ""); } diff --git a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java index 99e075a..a821b25 100644 --- a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java +++ b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java @@ -153,7 +153,7 @@ public abstract class BasicTestSetup extends AtlasTestBase { createClassificationTypes(); - AtlasEntity salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales"); + AtlasEntity salesDB = database("Sales", "/apps/warehouse/Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales"); entities.add(salesDB); AtlasEntity sd = @@ -193,7 +193,7 @@ public abstract class BasicTestSetup extends AtlasTestBase { entities.add(timeDim); AtlasEntity reportingDB = - database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting"); + database("Reporting", "/apps/warehouse/reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting"); entities.add(reportingDB); sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id"))); @@ -229,7 +229,7 @@ public abstract class BasicTestSetup extends AtlasTestBase { ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", ETL_CLASSIFICATION); entities.add(loadSalesDaily); - AtlasEntity logDB = database("Logging", null, "Tim ETL", "hdfs://host:8000/apps/warehouse/logging"); + AtlasEntity logDB = database("Logging", "/apps/warehouse/logging", "Tim ETL", "hdfs://host:8000/apps/warehouse/logging"); entities.add(logDB); sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id"))); @@ -336,8 +336,9 @@ public abstract class BasicTestSetup extends AtlasTestBase { database.setAttribute("name", name); database.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "qualified:" + name); database.setAttribute("description", description); + database.setAttribute("userDescription", description); database.setAttribute("owner", owner); - database.setAttribute("locationUri", locationUri); + database.setAttribute("location", locationUri); database.setAttribute("createTime", System.currentTimeMillis()); database.setAttribute("clusterName", "cl1"); database.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList())); diff --git a/repository/src/test/java/org/apache/atlas/query/BaseDSLComposer.java b/repository/src/test/java/org/apache/atlas/query/BaseDSLComposer.java index b104001..419050e 100644 --- a/repository/src/test/java/org/apache/atlas/query/BaseDSLComposer.java +++ b/repository/src/test/java/org/apache/atlas/query/BaseDSLComposer.java @@ -166,6 +166,8 @@ public class BaseDSLComposer { (context.getActiveTypeName().equals("hive_db") && attributeName.equals("name")) || (context.getActiveTypeName().equals("hive_db") && attributeName.equals("owner")) || (context.getActiveTypeName().equals("hive_db") && attributeName.equals("createTime")) || + (context.getActiveTypeName().equals("hive_db") && attributeName.equals("description")) || + (context.getActiveTypeName().equals("hive_db") && attributeName.equals("userDescription")) || (context.getActiveTypeName().equals("DB") && attributeName.equals("name")) || (context.getActiveTypeName().equals("DB") && attributeName.equals("owner")) || (context.getActiveTypeName().equals("DB") && attributeName.equals("clusterName")) || diff --git a/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java b/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java index 0c1dd59..3404dc6 100644 --- a/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java +++ b/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java @@ -561,6 +561,12 @@ public class DSLQueriesTest extends BasicTestSetup { {"hive_table where name like 'sales*' and db.name like 'Sa?es'", 1, new ListValidator("sales_fact")}, {"hive_table where db.name like \"Sa*\"", 4, new ListValidator("customer_dim", "sales_fact", "time_dim", "product_dim")}, {"hive_table where db.name like \"Sa*\" and name like \"*dim\"", 3, new ListValidator("customer_dim", "product_dim", "time_dim")}, + //STRING Mapping + {"hive_db where userDescription like \"*/warehouse/*\"", 3, new ListValidator("Sales","Reporting","Logging")}, + {"hive_db where userDescription like \"/apps/warehouse/*\"", 3, new ListValidator("Sales","Reporting","Logging")}, + //TEXT Mapping + {"hive_db where description like \"*/warehouse/*\"", 3, new ListValidator("Sales","Reporting","Logging")}, + {"hive_db where description like \"/apps/warehouse/*\"", 3, new ListValidator("Sales","Reporting","Logging")}, }; } diff --git a/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java b/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java index 3a2d66d..6220c23 100644 --- a/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java +++ b/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java @@ -197,6 +197,9 @@ public class GremlinQueryComposerTest { "g.V().has('__typeName', 'Table').has('Table.owner', org.janusgraph.core.attribute.Text.textRegex(\".*Tab_.*\")).dedup().limit(25).toList()"); verify("from Table where (db.name = \"Reporting\")", "g.V().has('__typeName', 'Table').out('__Table.db').has('DB.name', eq(\"Reporting\")).dedup().in('__Table.db').dedup().limit(25).toList()"); + verify( "Table where owner like \"Jane/*\"", "g.V().has('__typeName', 'Table').has('Table.owner', org.janusgraph.core.attribute.Text.textRegex(\"Jane\\/.*\")).dedup().limit(25).toList()"); + verify( "Table where Asset.name like \"/sales_*\"", "g.V().has('__typeName', 'Table').has('Asset.__s_name', org.janusgraph.core.attribute.Text.textRegex(\"\\/sales_.*\")).dedup().limit(25).toList()"); + verify( "Table where Asset.name like \"sales:*\"", "g.V().has('__typeName', 'Table').has('Asset.__s_name', org.janusgraph.core.attribute.Text.textRegex(\"sales:.*\")).dedup().limit(25).toList()"); } @Test