Repository: incubator-atlas Updated Branches: refs/heads/master 19d344cd8 -> ca64ef4e8
ATLAS-1742: Provide option to exclude deleted entities in basic and fulltext search Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ca64ef4e Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ca64ef4e Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ca64ef4e Branch: refs/heads/master Commit: ca64ef4e80fe621435b93888abb2feee712a6283 Parents: 19d344c Author: Sarath Subramanian <[email protected]> Authored: Fri Apr 28 17:42:02 2017 -0700 Committer: Sarath Subramanian <[email protected]> Committed: Fri Apr 28 17:42:02 2017 -0700 ---------------------------------------------------------------------- .../atlas/discovery/AtlasDiscoveryService.java | 6 ++-- .../atlas/discovery/EntityDiscoveryService.java | 34 ++++++++++++++++---- .../apache/atlas/web/rest/DiscoveryREST.java | 27 +++++++++------- 3 files changed, 46 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ca64ef4e/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index 30d62a4..923a198 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -35,11 +35,12 @@ public interface AtlasDiscoveryService { /** * * @param query search query. + * @param excludeDeletedEntities exclude deleted entities in search result. * @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property. * @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0. * @return AtlasSearchResult */ - AtlasSearchResult searchUsingFullTextQuery(String query, int limit, int offset) throws AtlasBaseException; + AtlasSearchResult searchUsingFullTextQuery(String query, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException; /** * @@ -48,10 +49,11 @@ public interface AtlasDiscoveryService { * @param classification classification name. * @param attrName attribute name. * @param attrValuePrefix attribute value prefix. + * @param excludeDeletedEntities exclude deleted entities in search result. * @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property. * @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0. * @return AtlasSearchResult */ AtlasSearchResult searchUsingBasicQuery(String query, String type, String classification, String attrName, - String attrValuePrefix, int limit, int offset) throws AtlasBaseException; + String attrValuePrefix, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ca64ef4e/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 07190db..f3f6ee2 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -25,6 +25,7 @@ import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult; import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.instance.AtlasEntity.Status; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.query.Expressions.AliasExpression; import org.apache.atlas.query.Expressions.Expression; @@ -46,7 +47,6 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever; import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.AtlasGremlinQueryProvider; @@ -145,7 +145,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } @Override - public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, int limit, int offset) throws AtlasBaseException { + public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, boolean excludeDeletedEntities, int limit, int offset) + throws AtlasBaseException { AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT); QueryParams params = validateSearchParams(limit, offset); AtlasIndexQuery idxQuery = toAtlasIndexQuery(fullTextQuery); @@ -153,14 +154,16 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { if (LOG.isDebugEnabled()) { LOG.debug("Executing Full text query: {}", fullTextQuery); } - ret.setFullTextResult(getIndexQueryResults(idxQuery, params)); + ret.setFullTextResult(getIndexQueryResults(idxQuery, params, excludeDeletedEntities)); return ret; } @Override public AtlasSearchResult searchUsingBasicQuery(String query, String typeName, String classification, String attrName, - String attrValuePrefix, int limit, int offset) throws AtlasBaseException { + String attrValuePrefix, boolean excludeDeletedEntities, int limit, + int offset) throws AtlasBaseException { + AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.BASIC); if (LOG.isDebugEnabled()) { @@ -267,6 +270,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } } + if (skipDeletedEntities(excludeDeletedEntities, vertex)) { + continue; + } + resultIdx++; if (resultIdx <= startIdx) { @@ -314,6 +321,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { if (firstElement instanceof AtlasVertex) { for (Object element : queryResult) { if (element instanceof AtlasVertex) { + if (skipDeletedEntities(excludeDeletedEntities, (AtlasVertex) element)) { + continue; + } + ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex) element)); } else { LOG.warn("searchUsingBasicQuery({}): expected an AtlasVertex; found unexpected entry in result {}", basicQuery, element); @@ -331,13 +342,18 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return ret; } - private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams params) throws AtlasBaseException { + private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams params, boolean excludeDeletedEntities) throws AtlasBaseException { List<AtlasFullTextResult> ret = new ArrayList<>(); Iterator<Result> iter = query.vertices(); while (iter.hasNext() && ret.size() < params.limit()) { - Result idxQueryResult = iter.next(); - AtlasVertex vertex = idxQueryResult.getVertex(); + Result idxQueryResult = iter.next(); + AtlasVertex vertex = idxQueryResult.getVertex(); + + if (skipDeletedEntities(excludeDeletedEntities, vertex)) { + continue; + } + String guid = vertex != null ? vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class) : null; if (guid != null) { @@ -422,4 +438,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return ret; } + + private boolean skipDeletedEntities(boolean excludeDeletedEntities, AtlasVertex<?, ?> vertex) { + return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED; + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ca64ef4e/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 76ad88c..81dbe25 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -114,6 +114,7 @@ public class DiscoveryREST { @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) public AtlasSearchResult searchUsingFullText(@QueryParam("query") String query, + @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities, @QueryParam("limit") int limit, @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; @@ -124,7 +125,7 @@ public class DiscoveryREST { limit + "," + offset + ")"); } - return atlasDiscoveryService.searchUsingFullTextQuery(query, limit, offset); + return atlasDiscoveryService.searchUsingFullTextQuery(query, excludeDeletedEntities, limit, offset); } finally { AtlasPerfTracer.log(perf); } @@ -147,11 +148,12 @@ public class DiscoveryREST { @Path("/basic") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasSearchResult searchUsingBasic(@QueryParam("query") String query, - @QueryParam("typeName") String typeName, - @QueryParam("classification") String classification, - @QueryParam("limit") int limit, - @QueryParam("offset") int offset) throws AtlasBaseException { + public AtlasSearchResult searchUsingBasic(@QueryParam("query") String query, + @QueryParam("typeName") String typeName, + @QueryParam("classification") String classification, + @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities, + @QueryParam("limit") int limit, + @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -160,7 +162,8 @@ public class DiscoveryREST { typeName + "," + classification + "," + limit + "," + offset + ")"); } - return atlasDiscoveryService.searchUsingBasicQuery(query, typeName, classification, null, null, limit, offset); + return atlasDiscoveryService.searchUsingBasicQuery(query, typeName, classification, null, null, + excludeDeletedEntities, limit, offset); } finally { AtlasPerfTracer.log(perf); } @@ -183,11 +186,11 @@ public class DiscoveryREST { @Path("/attribute") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasSearchResult searchUsingAttribute(@QueryParam("attrName") String attrName, + public AtlasSearchResult searchUsingAttribute(@QueryParam("attrName") String attrName, @QueryParam("attrValuePrefix") String attrValuePrefix, - @QueryParam("typeName") String typeName, - @QueryParam("limit") int limit, - @QueryParam("offset") int offset) throws AtlasBaseException { + @QueryParam("typeName") String typeName, + @QueryParam("limit") int limit, + @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -201,7 +204,7 @@ public class DiscoveryREST { String.format("attrName : {0}, attrValue: {1} for attribute search.", attrName, attrValuePrefix)); } - return atlasDiscoveryService.searchUsingBasicQuery(null, typeName, null, attrName, attrValuePrefix, limit, offset); + return atlasDiscoveryService.searchUsingBasicQuery(null, typeName, null, attrName, attrValuePrefix, true, limit, offset); } finally { AtlasPerfTracer.log(perf);
