Repository: atlas Updated Branches: refs/heads/branch-1.0 3d69ad2f4 -> fb32d155a
ATLAS-2775: Enhance basic search to search on entities with any classifications or no classifications (cherry picked from commit 15388229741b19c4567527da11bbb1daf2f9399f) Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/fb32d155 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/fb32d155 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/fb32d155 Branch: refs/heads/branch-1.0 Commit: fb32d155a2a6c196a0bb8ca65b20aa4754f70bfb Parents: 3d69ad2 Author: Sarath Subramanian <ssubraman...@hortonworks.com> Authored: Tue Jul 3 14:44:37 2018 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Tue Jul 3 15:03:08 2018 -0700 ---------------------------------------------------------------------- .../atlas/model/discovery/SearchParameters.java | 4 ++ .../ClassificationSearchProcessor.java | 39 ++++++++++++++------ .../atlas/discovery/EntitySearchProcessor.java | 32 +++++++++++----- .../discovery/FullTextSearchProcessor.java | 8 +++- .../apache/atlas/discovery/SearchContext.java | 16 ++++++-- 5 files changed, 73 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/fb32d155/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java index 39ebf23..aac6b5a 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java @@ -55,6 +55,10 @@ public class SearchParameters implements Serializable { private FilterCriteria tagFilters; private Set<String> attributes; + public static final String WILDCARD_CLASSIFICATIONS = "*"; + public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED"; + public static final String NO_CLASSIFICATIONS = "_NOT_CLASSIFIED"; + /** * @return The type of query */ http://git-wip-us.apache.org/repos/asf/atlas/blob/fb32d155/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java index 588ad3e..c00bfb6 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java @@ -50,6 +50,12 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION; +import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL; @@ -88,7 +94,9 @@ public class ClassificationSearchProcessor extends SearchProcessor { processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes); // for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query - boolean useIndexSearch = classificationType != SearchContext.MATCH_ALL_CLASSIFICATION && + boolean useIndexSearch = classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION && + classificationType != MATCH_ALL_CLASSIFIED && + classificationType != MATCH_ALL_NOT_CLASSIFIED && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS && CollectionUtils.isEmpty(graphAttributes) && canApplyIndexFilter(classificationType, filterCriteria, false); @@ -149,25 +157,32 @@ public class ClassificationSearchProcessor extends SearchProcessor { LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery); } } else { - tagGraphQueryWithAttributes = null; - List<AtlasGraphQuery> orConditions = new LinkedList<>(); + tagGraphQueryWithAttributes = null; + List<AtlasGraphQuery> orConditions = new LinkedList<>(); - if (classificationType != SearchContext.MATCH_ALL_CLASSIFICATION) { - orConditions.add(graph.query().createChildQuery().in(Constants.TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes)); - orConditions.add(graph.query().createChildQuery().in(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes)); + if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) { + orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); + orConditions.add(graph.query().createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); entityGraphQueryTraitNames = graph.query().or(orConditions); entityPredicateTraitNames = PredicateUtils.orPredicate( - SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class), - SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class)); + SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), + SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); + } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { + orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null).has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null)); + + entityGraphQueryTraitNames = graph.query().or(orConditions); + entityPredicateTraitNames = PredicateUtils.andPredicate( + SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), + SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); } else { - orConditions.add(graph.query().createChildQuery().has(Constants.TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); - orConditions.add(graph.query().createChildQuery().has(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); + orConditions.add(graph.query().createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes)); + orConditions.add(graph.query().createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes)); entityGraphQueryTraitNames = graph.query().or(orConditions); entityPredicateTraitNames = PredicateUtils.orPredicate( - SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, null, List.class), - SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); + SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class), + SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class)); } if (context.getSearchParameters().getExcludeDeletedEntities()) { http://git-wip-us.apache.org/repos/asf/atlas/blob/fb32d155/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java index abe859f..c4b5ffb 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java @@ -40,6 +40,12 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION; +import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL; public class EntitySearchProcessor extends SearchProcessor { @@ -93,12 +99,15 @@ public class EntitySearchProcessor extends SearchProcessor { .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class); final Predicate traitPredicate; - if (classificationType == SearchContext.MATCH_ALL_CLASSIFICATION) { - traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, null, List.class), - SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); + if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) { + traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), + SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); + } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { + traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), + SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); } else { - traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class), - SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class)); + traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class), + SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class)); } processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes); @@ -154,12 +163,15 @@ public class EntitySearchProcessor extends SearchProcessor { if (filterClassification) { List<AtlasGraphQuery> orConditions = new LinkedList<>(); - if (classificationType == SearchContext.MATCH_ALL_CLASSIFICATION) { - orConditions.add(query.createChildQuery().has(Constants.TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); - orConditions.add(query.createChildQuery().has(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); + if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) { + orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); + orConditions.add(query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); + } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { + orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null) + .has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null)); } else { - orConditions.add(query.createChildQuery().in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes)); - orConditions.add(query.createChildQuery().in(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes)); + orConditions.add(query.createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes)); + orConditions.add(query.createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes)); } query.or(orConditions); http://git-wip-us.apache.org/repos/asf/atlas/blob/fb32d155/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java index 2c84369..0ffd61c 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java @@ -31,6 +31,10 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION; + public class FullTextSearchProcessor extends SearchProcessor { private static final Logger LOG = LoggerFactory.getLogger(FullTextSearchProcessor.class); @@ -61,7 +65,9 @@ public class FullTextSearchProcessor extends SearchProcessor { // if search includes classification criteria, adding a filter here can help avoid unnecessary // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor - if (context.getClassificationType() != null && context.getClassificationType() != SearchContext.MATCH_ALL_CLASSIFICATION) { + if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION && + context.getClassificationType() != MATCH_ALL_CLASSIFIED && + context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) { String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr(); if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) { http://git-wip-us.apache.org/repos/asf/atlas/blob/fb32d155/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java index 1f40076..b4542b7 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java @@ -45,6 +45,10 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS; +import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS; +import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS; + /* * Search context captures elements required for performing a basic search * For every search request the search context will determine the execution sequence of the search processor(s) and the @@ -61,7 +65,9 @@ public class SearchContext { private SearchProcessor searchProcessor; private boolean terminateSearch = false; - public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef("*")); + public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS)); + public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS)); + public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS)); public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException { String classificationName = searchParameters.getClassification(); @@ -199,8 +205,12 @@ public class SearchContext { private AtlasClassificationType getClassificationType(String classificationName) { AtlasClassificationType ret; - if (StringUtils.equals(classificationName, MATCH_ALL_CLASSIFICATION.getTypeName())) { - ret = MATCH_ALL_CLASSIFICATION; + if (StringUtils.equals(classificationName, MATCH_ALL_WILDCARD_CLASSIFICATION.getTypeName())) { + ret = MATCH_ALL_WILDCARD_CLASSIFICATION; + } else if (StringUtils.equals(classificationName, MATCH_ALL_CLASSIFIED.getTypeName())) { + ret = MATCH_ALL_CLASSIFIED; + } else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) { + ret = MATCH_ALL_NOT_CLASSIFIED; } else { ret = typeRegistry.getClassificationTypeByName(classificationName); }
