Repository: usergrid Updated Branches: refs/heads/master cf0b65366 -> 447b5b830
Enhance the elasticsearch filter so that sort predicates of known entity fields have a sort applied with their specific type rather than all possible types. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/447b5b83 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/447b5b83 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/447b5b83 Branch: refs/heads/master Commit: 447b5b8305f6fddb8ead78755eb0bbeab5dd9ca7 Parents: cf0b653 Author: Michael Russo <russomich...@google.com> Authored: Wed Feb 22 10:43:16 2017 -0800 Committer: Michael Russo <russomich...@google.com> Committed: Wed Feb 22 10:43:16 2017 -0800 ---------------------------------------------------------------------- .../search/AbstractElasticSearchFilter.java | 20 ++++++++++++- .../usergrid/persistence/index/EntityIndex.java | 16 ++++++++++ .../index/impl/EsEntityIndexImpl.java | 8 ++++- .../persistence/index/impl/EsQueryVistor.java | 26 ++++++++++++++++ .../impl/SearchRequestBuilderStrategy.java | 31 ++++++++++---------- 5 files changed, 83 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/447b5b83/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java index deba16a..4bf723e 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java @@ -21,6 +21,7 @@ package org.apache.usergrid.corepersistence.pipeline.read.search; import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory; +import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.index.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +39,8 @@ import com.google.common.base.Optional; import rx.Observable; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; /** @@ -80,6 +83,21 @@ public abstract class AbstractElasticSearchFilter extends AbstractPathFilter<Id, final SearchTypes searchTypes = getSearchTypes(); + // pull out the basic Usergrid entity info to get known properties and their associated types + final Map<String, Class> propertiesWithType = new HashMap<>(); + for (String type : searchTypes.getTypes()) { + try { + if ( Schema.getDefaultSchema().getEntityInfo(type) != null ){ + Schema.getDefaultSchema().getEntityInfo(type).getProperties().forEach((propName, propValue) -> + propertiesWithType.put(propName, propValue.getType()) + ); + } + }catch (Exception e){ + // do nothing here, clear the potentially partially filled map and fall back to original behavior + propertiesWithType.clear(); + logger.warn("Unable to obtain the default entity type fields with type. Sort may have degraded performance."); + } + } //return all ids that are emitted from this edge return observable.flatMap( idFilterResult -> { @@ -105,7 +123,7 @@ public abstract class AbstractElasticSearchFilter extends AbstractPathFilter<Id, try { final CandidateResults candidateResults = - applicationEntityIndex.search( searchEdge, searchTypes, query, limit, currentOffSet ); + applicationEntityIndex.search( searchEdge, searchTypes, query, limit, currentOffSet, propertiesWithType ); Collection<SelectFieldMapping> fieldMappingCollection = candidateResults.getGetFieldMappings(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/447b5b83/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index 8ab2d41..ee12ec8 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@ -26,6 +26,7 @@ import org.apache.usergrid.persistence.core.util.Health; import org.apache.usergrid.persistence.model.entity.Id; import rx.Observable; +import java.util.Map; import java.util.UUID; @@ -102,6 +103,21 @@ public interface EntityIndex extends CPManager { CandidateResults search(final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, final int limit, final int offset); + /** + * Search on every document in the specified search edge. Also search by the types if specified + * + * @param searchEdge The edge to search on + * @param searchTypes The search types to search + * @param query The query to execute + * @param limit The limit of values to return + * @param offset The offset to query on + * @param fieldsWithType An optional param that allows the caller to provide schema related info which might + * relate to data in the query, such as sort predicate types + * @return + */ + CandidateResults search(final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, + final int limit, final int offset, final Map<String, Class> fieldsWithType); + /** * Same as search, just iterates all documents that match the index edge exactly. http://git-wip-us.apache.org/repos/asf/usergrid/blob/447b5b83/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 6e04bed..0b7f18d 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -411,6 +411,11 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, final int limit, final int offset ) { + return search(searchEdge, searchTypes, query, limit, offset, new HashMap<>(0)); + } + + public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, + final int limit, final int offset, final Map<String, Class> fieldsWithType ) { IndexValidationUtils.validateSearchEdge(searchEdge); Preconditions.checkNotNull(searchTypes, "searchTypes cannot be null"); @@ -422,7 +427,8 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { final ParsedQuery parsedQuery = ParsedQueryBuilder.build(query); - final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes, parsedQuery, limit, offset ) + final SearchRequestBuilder srb = searchRequest + .getBuilder( searchEdge, searchTypes, parsedQuery, limit, offset, fieldsWithType ) .setTimeout(TimeValue.timeValueMillis(queryTimeout)); if ( logger.isDebugEnabled() ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/447b5b83/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java index 414f716..7ee8961 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java @@ -526,6 +526,32 @@ public class EsQueryVistor implements QueryVisitor { "Unkown search type of " + object.getClass().getName() + " encountered" ); } + /** + * Get the field name for the primitive type + */ + public static String getFieldNameForClass( final Class clazz ) { + if ( clazz == String.class || clazz == UUID.class ) { + return IndexingUtils.FIELD_STRING_NESTED; + } + + if ( clazz == Boolean.class ) { + return IndexingUtils.FIELD_BOOLEAN_NESTED; + } + + + if ( clazz == Integer.class || clazz == Long.class ) { + return IndexingUtils.FIELD_LONG_NESTED; + } + + if ( clazz == Float.class || clazz == Double.class ) { + return IndexingUtils.FIELD_DOUBLE_NESTED; + } + + + throw new UnsupportedOperationException( + "Unkown search type of " + clazz.getClass().getName() + " encountered" ); + } + /** * Lowercase our input http://git-wip-us.apache.org/repos/asf/usergrid/blob/447b5b83/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java index a5c1af3..c201fb6 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java @@ -47,6 +47,8 @@ import org.apache.usergrid.persistence.index.query.tree.QueryVisitor; import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import java.util.Map; + import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName; import static org.apache.usergrid.persistence.index.impl.SortBuilder.sortPropertyTermFilter; @@ -79,7 +81,8 @@ public class SearchRequestBuilderStrategy { * Get the search request builder */ public SearchRequestBuilder getBuilder( final SearchEdge searchEdge, final SearchTypes searchTypes, - final ParsedQuery query, final int limit, final int from ) { + final ParsedQuery query, final int limit, final int from, + final Map<String, Class> fieldsWithType ) { Preconditions .checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than max " + EntityIndex.MAX_LIMIT ); @@ -112,7 +115,7 @@ public class SearchRequestBuilderStrategy { applyDefaultSortPredicates( srb, geoFields ); } else { - applySortPredicates( srb, query, geoFields ); + applySortPredicates( srb, query, geoFields, fieldsWithType ); } @@ -147,44 +150,40 @@ public class SearchRequestBuilderStrategy { * Invoked when there are sort predicates */ private void applySortPredicates( final SearchRequestBuilder srb, final ParsedQuery query, - final GeoSortFields geoFields ) { - + final GeoSortFields geoFields, final Map<String, Class> knownFieldsWithType ) { //we have sort predicates, sort them for ( SortPredicate sp : query.getSortPredicates() ) { - - // we do not know the type of the "order by" property and so we do not know what - // type prefix to use. So, here we add an order by clause for every possible type - // that you can order by: string, number and boolean and we ask ElasticSearch - // to ignore any fields that are not present. final SortOrder order = sp.getDirection().toEsSort(); final String propertyName = sp.getPropertyName(); - //if the user specified a geo field in their sort, then honor their sort order and use the point they // specified if ( geoFields.contains( propertyName ) ) { final GeoDistanceSortBuilder geoSort = geoFields.applyOrder( propertyName, SortOrder.ASC ); srb.addSort( geoSort ); } - - //apply regular sort logic, since this is not a geo point + // fieldsWithType gives the caller an option to provide any schema related details on properties that + // might appear in a sort predicate. loop through these and set a specific sort, rather than adding a sort + // for all possible types + else if ( knownFieldsWithType != null && knownFieldsWithType.size() > 0) { + if (knownFieldsWithType.containsKey(propertyName)){ + srb.addSort( createSort( order, EsQueryVistor.getFieldNameForClass(knownFieldsWithType.get(propertyName)), propertyName ) ); + } + } + //apply regular sort logic which check all possible data types, since this is not a known property name else { - //sort order is arbitrary if the user changes data types. Double, long, string, boolean are supported //default sort types - srb.addSort( createSort( order, IndexingUtils.FIELD_DOUBLE_NESTED, propertyName ) ); - srb.addSort( createSort( order, IndexingUtils.FIELD_LONG_NESTED, propertyName ) ); /** * We always want to sort by the unanalyzed string field to ensure correct ordering */ srb.addSort( createSort( order, IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, propertyName ) ); - srb.addSort( createSort( order, IndexingUtils.FIELD_BOOLEAN_NESTED, propertyName ) ); } }
