Repository: usergrid Updated Branches: refs/heads/3.0.0_experimental 8b945c638 -> 697b291ec
Upgrade Elastic search t0 5.6.3 Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bb593d46 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bb593d46 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bb593d46 Branch: refs/heads/3.0.0_experimental Commit: bb593d463734a0161dbecb61c17a69ee5363273c Parents: b93f8d4 Author: Peter Johnson <pjohn...@apigee.com> Authored: Tue Oct 24 16:07:37 2017 -0700 Committer: Peter Johnson <pjohn...@apigee.com> Committed: Tue Oct 24 17:14:13 2017 -0700 ---------------------------------------------------------------------- stack/corepersistence/pom.xml | 2 +- stack/corepersistence/queryindex/pom.xml | 6 ++ .../index/impl/EsEntityIndexImpl.java | 66 ++++++------ .../index/impl/EsIndexProducerImpl.java | 23 ++++- .../persistence/index/impl/EsProvider.java | 18 ++-- .../persistence/index/impl/EsQueryVistor.java | 100 ++++++++----------- .../index/impl/NoOpFilterBuilder.java | 13 +-- .../index/impl/NoOpQueryBuilder.java | 33 +++++- .../impl/SearchRequestBuilderStrategy.java | 36 ++++--- .../persistence/index/impl/SortBuilder.java | 8 +- .../index/query/tree/QueryVisitor.java | 10 +- .../persistence/index/usergrid-mappings.json | 2 - 12 files changed, 174 insertions(+), 143 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/pom.xml ---------------------------------------------------------------------- diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml index 45aad4b..ff09cf2 100644 --- a/stack/corepersistence/pom.xml +++ b/stack/corepersistence/pom.xml @@ -82,7 +82,7 @@ limitations under the License. <commons.io.version>2.4</commons.io.version> <commons.lang.version>3.1</commons.lang.version> <datastax.version>2.1.10.3</datastax.version> - <elasticsearch.version>1.7.5</elasticsearch.version> + <elasticsearch.version>5.6.3</elasticsearch.version> <fasterxml-uuid.version>3.1.3</fasterxml-uuid.version> <guava.version>18.0</guava.version> <guice.version>4.0-beta5</guice.version> http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/pom.xml ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml index 11856cb..c5401e4 100644 --- a/stack/corepersistence/queryindex/pom.xml +++ b/stack/corepersistence/queryindex/pom.xml @@ -101,6 +101,12 @@ <type>jar</type> </dependency> + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>transport</artifactId> + <version>${elasticsearch.version}</version> + </dependency> + <dependency> <groupId>org.elasticsearch</groupId> http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 3d2f576..c5b5ec4 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -47,10 +47,8 @@ import org.apache.usergrid.persistence.index.query.tree.QueryVisitor; import org.apache.usergrid.persistence.index.utils.IndexValidationUtils; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.util.UUIDGenerator; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.action.*; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; @@ -59,22 +57,21 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; -import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.AdminClient; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.indices.IndexAlreadyExistsException; -import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder; +import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -206,13 +203,13 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { //Create index try { final AdminClient admin = esProvider.getClient().admin(); - Settings settings = ImmutableSettings.settingsBuilder() + Settings settings = Settings.builder() .put("index.number_of_shards", numberOfShards) .put("index.number_of_replicas", numberOfReplicas) //dont' allow unmapped queries, and don't allow dynamic mapping .put("index.query.parse.allow_unmapped_fields", false) .put("index.mapper.dynamic", false) - .put("action.write_consistency", writeConsistency) + // .put("action.write_consistency", writeConsistency) .build(); //Added For Graphite Metrics @@ -230,7 +227,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged()); - } catch (IndexAlreadyExistsException e) { + } catch (ResourceAlreadyExistsException e) { logger.info("Index Name [{}] already exists", indexName); } /** @@ -244,7 +241,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { testNewIndex(); - } catch (IndexAlreadyExistsException expected) { + } catch (ResourceAlreadyExistsException expected) { // this is expected to happen if index already exists, it's a no-op and swallow } catch (IOException e) { throw new RuntimeException("Unable to initialize index", e); @@ -452,8 +449,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { for (SortPredicate sortPredicate : parsedQuery.getSortPredicates() ){ hasGeoSortPredicates = visitor.getGeoSorts().contains(sortPredicate.getPropertyName()); } - - + final String cacheKey = applicationScope.getApplication().getUuid().toString()+"_"+searchEdge.getEdgeName(); final Object totalEdgeSizeFromCache = sizeCache.getIfPresent(cacheKey); long totalEdgeSizeInBytes; @@ -498,6 +494,9 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { final Timer.Context timerContext = searchTimer.time(); try { + if (logger.isDebugEnabled()) { + logger.debug("Query to execute = {}", srb.toString()); + } searchResponse = srb.execute().actionGet(); } @@ -594,21 +593,25 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { //Added For Graphite Metrics return Observable.from( indexes ).flatMap( index -> { - final ListenableActionFuture<DeleteByQueryResponse> response = - esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute(); + ListenableActionFuture<BulkByScrollResponse> response = + DeleteByQueryAction.INSTANCE.newRequestBuilder( esProvider.getClient()) + .filter(tqb) + .source(indexes) + .execute(); + - response.addListener( new ActionListener<DeleteByQueryResponse>() { + response.addListener( new ActionListener<BulkByScrollResponse>() { @Override - public void onResponse( DeleteByQueryResponse response ) { + public void onResponse( BulkByScrollResponse response ) { checkDeleteByQueryResponse( tqb, response ); } - @Override - public void onFailure( Throwable e ) { + public void onFailure(Exception e) { logger.error( "Failed on delete index", e.getMessage() ); } + } ); return Observable.from( response ); } ).doOnError( t -> logger.error( "Failed on delete application", t.getMessage() ) ); @@ -618,17 +621,14 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { /** * Validate the response doesn't contain errors, if it does, fail fast at the first error we encounter */ - private void checkDeleteByQueryResponse( final QueryBuilder query, final DeleteByQueryResponse response ) { + private void checkDeleteByQueryResponse( final QueryBuilder query, final BulkByScrollResponse response ) { - for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) { - final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures(); + List<BulkItemResponse.Failure> failures = response.getBulkFailures(); - for ( ShardOperationFailedException failedException : failures ) { - logger.error("Unable to delete by query {}. Failed with code {} and reason {} on shard {} in index {}", - query.toString(), - failedException.status().getStatus(), failedException.reason(), - failedException.shardId(), failedException.index() ); - } + + for ( BulkItemResponse.Failure failure : failures ) { + logger.error("Unable to delete by query {}. Failed with code {} and reason {} in index {}", + query.toString(), failure.getStatus() , failure.getMessage(), failure.getIndex()); } } @@ -817,7 +817,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { .actionGet(); final CommonStats indexStats = statsResponse.getIndex(indexName).getTotal(); indexSize = indexStats.getStore().getSizeInBytes(); - } catch (IndexMissingException e) { + } catch (IndexNotFoundException e) { // if for some reason the index size does not exist, // log an error and we can assume size is 0 as it doesn't exist logger.error("Unable to get size for index {} due to IndexMissingException for app {}", @@ -836,7 +836,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { private long getEntitySizeAggregation( final SearchRequestBuilder builder ) { final String key = "entitySize"; - SumBuilder sumBuilder = new SumBuilder(key); + SumAggregationBuilder sumBuilder = new SumAggregationBuilder(key); sumBuilder.field("entitySize"); builder.addAggregation(sumBuilder); http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java index 8f58ef7..468a99e 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java @@ -26,10 +26,10 @@ import com.codahale.metrics.Histogram; import org.apache.usergrid.persistence.index.EntityIndexBatch; -import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.Client; import org.elasticsearch.rest.RestStatus; import org.slf4j.Logger; @@ -169,9 +169,26 @@ public class EsIndexProducerImpl implements IndexProducer { * initialize request */ private BulkRequestBuilder initRequest() { + BulkRequestBuilder bulkRequest = client.prepareBulk(); - bulkRequest.setConsistencyLevel( WriteConsistencyLevel.fromString( config.getWriteConsistencyLevel() ) ); - bulkRequest.setRefresh( config.isForcedRefresh() ); + + String refreshPolicyConfig = String.valueOf(config.isForcedRefresh()); + bulkRequest.setRefreshPolicy(refreshPolicyConfig); + + String consistencyLevel = config.getWriteConsistencyLevel(); + + if ("one".equals(consistencyLevel)) { + bulkRequest.setWaitForActiveShards(1); + } + + if ("all".equals(consistencyLevel)) { + bulkRequest.setWaitForActiveShards(ActiveShardCount.ALL); + } + + if ("none".equals(consistencyLevel)) { + bulkRequest.setWaitForActiveShards(ActiveShardCount.NONE); + } + return bulkRequest; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java index e81aa0f..fe35a02 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java @@ -24,11 +24,10 @@ import java.net.UnknownHostException; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,7 +122,8 @@ public class EsProvider { final String clusterName = indexFig.getClusterName(); final int port = indexFig.getPort(); - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put( "cluster.name", clusterName ) + Settings.Builder settings = Settings.builder() + .put( "cluster.name", clusterName ) .put( "client.transport.sniff", true ); String nodeName = indexFig.getNodeName(); @@ -142,11 +142,15 @@ public class EsProvider { settings.put( "node.name", nodeName ); - TransportClient transportClient = new TransportClient( settings.build() ); + TransportClient transportClient = new PreBuiltTransportClient(settings.build()); // we will connect to ES on all configured hosts for ( String host : indexFig.getHosts().split( "," ) ) { - transportClient.addTransportAddress( new InetSocketTransportAddress( host, port ) ); + try { + transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port)); + } catch (UnknownHostException uhe) { + logger.error( "Couldn't resolve hostname {} to use as ES node ", host ); + } } return transportClient; @@ -181,7 +185,7 @@ public class EsProvider { final String hostString = hosts.toString(); - Settings settings = ImmutableSettings.settingsBuilder() + Settings settings = Settings.builder() .put( "cluster.name", clusterName ) @@ -200,7 +204,7 @@ public class EsProvider { logger.trace("Creating ElasticSearch client with settings: {}", settings.getAsMap()); } - Node node = NodeBuilder.nodeBuilder().settings( settings ).client( true ).data( false ).node(); + Node node = new Node( settings ); return node.client(); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java index 4dd0d24..7adb332 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java @@ -22,20 +22,11 @@ package org.apache.usergrid.persistence.index.impl; import java.util.Stack; import java.util.UUID; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.index.query.BoolFilterBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.NestedFilterBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.RangeFilterBuilder; -import org.elasticsearch.index.query.TermFilterBuilder; -import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.index.query.*; + import org.elasticsearch.search.sort.GeoDistanceSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.slf4j.Logger; @@ -75,7 +66,7 @@ public class EsQueryVistor implements QueryVisitor { /** * Our queryBuilders for filter operations */ - private final Stack<FilterBuilder> filterBuilders = new Stack<>(); + private final Stack<QueryBuilder> filterBuilders = new Stack<>(); private final GeoSortFields geoSortFields = new GeoSortFields(); @@ -89,12 +80,12 @@ public class EsQueryVistor implements QueryVisitor { //get all the right final QueryBuilder rightQuery = queryBuilders.pop(); - final FilterBuilder rightFilter = filterBuilders.pop(); + final QueryBuilder rightFilter = filterBuilders.pop(); //get all the left final QueryBuilder leftQuery = queryBuilders.pop(); - final FilterBuilder leftFilter = filterBuilders.pop(); + final QueryBuilder leftFilter = filterBuilders.pop(); //push our boolean filters @@ -131,7 +122,7 @@ public class EsQueryVistor implements QueryVisitor { //use left and right if ( useLeftFilter && useRightFilter ) { - final BoolFilterBuilder fb = FilterBuilders.boolFilter().must(leftFilter).must(rightFilter); + final BoolQueryBuilder fb = QueryBuilders.boolQuery().must(leftFilter).must(rightFilter); filterBuilders.push( fb ); } @@ -157,12 +148,12 @@ public class EsQueryVistor implements QueryVisitor { op.getRight().visit( this ); final QueryBuilder rightQuery = queryBuilders.pop(); - final FilterBuilder rightFilter = filterBuilders.pop(); + final QueryBuilder rightFilter = filterBuilders.pop(); //get all the left final QueryBuilder leftQuery = queryBuilders.pop(); - final FilterBuilder leftFilter = filterBuilders.pop(); + final QueryBuilder leftFilter = filterBuilders.pop(); final boolean useLeftQuery = use( leftQuery ); @@ -193,7 +184,7 @@ public class EsQueryVistor implements QueryVisitor { //use left and right if ( useLeftFilter && useRightFilter ) { - final BoolFilterBuilder fb = FilterBuilders.boolFilter().should( leftFilter ).should( rightFilter ); + final BoolQueryBuilder fb = QueryBuilders.boolQuery().should( leftFilter ).should( rightFilter ); filterBuilders.push( fb ); } @@ -232,12 +223,12 @@ public class EsQueryVistor implements QueryVisitor { queryBuilders.push( NoOpQueryBuilder.INSTANCE ); } - final FilterBuilder notFilterBuilder = filterBuilders.pop(); + final QueryBuilder notFilterBuilder = filterBuilders.pop(); //push the filter in if ( use( notFilterBuilder ) ) { - final FilterBuilder notFilter = FilterBuilders.boolFilter().mustNot( notFilterBuilder ) ; + final QueryBuilder notFilter = QueryBuilders.boolQuery().mustNot( notFilterBuilder ) ; //just the root node if(!rootNode) { @@ -245,7 +236,7 @@ public class EsQueryVistor implements QueryVisitor { } //not the root node, we have to select all to subtract from with the NOT statement else{ - final FilterBuilder selectAllFilter = FilterBuilders.boolFilter().must( FilterBuilders.matchAllFilter()) .must( notFilter ); + final QueryBuilder selectAllFilter = QueryBuilders.boolQuery().must( QueryBuilders.matchAllQuery()) .must( notFilter ); filterBuilders.push( selectAllFilter ); } @@ -293,9 +284,10 @@ public class EsQueryVistor implements QueryVisitor { float distance = op.getDistance().getFloatValue(); - final FilterBuilder fb = - FilterBuilders.geoDistanceFilter( IndexingUtils.FIELD_LOCATION_NESTED ).lat( lat ).lon( lon ) - .distance( distance, DistanceUnit.METERS ); + final QueryBuilder fb = + QueryBuilders.geoDistanceQuery( IndexingUtils.FIELD_LOCATION_NESTED ) + .point(lat, lon) + .distance( distance, DistanceUnit.METERS ); filterBuilders.push( fieldNameTerm( name, fb ) ); @@ -306,10 +298,11 @@ public class EsQueryVistor implements QueryVisitor { //this geoSort won't has a sort on it final GeoDistanceSortBuilder geoSort = - SortBuilders.geoDistanceSort( IndexingUtils.FIELD_LOCATION_NESTED ).unit( DistanceUnit.METERS ) - .geoDistance(GeoDistance.SLOPPY_ARC).point(lat, lon); + SortBuilders.geoDistanceSort( IndexingUtils.FIELD_LOCATION_NESTED, lat, lon ) + .unit( DistanceUnit.METERS ) + .geoDistance(GeoDistance.ARC).point(lat, lon); - final TermFilterBuilder sortPropertyName = sortPropertyTermFilter(name); + final TermQueryBuilder sortPropertyName = sortPropertyTermFilter(name); geoSort.setNestedFilter( sortPropertyName ); @@ -328,8 +321,8 @@ public class EsQueryVistor implements QueryVisitor { final Object value = op.getLiteral().getValue(); - final RangeFilterBuilder termQuery = - FilterBuilders.rangeFilter( getFieldNameForType( value ) ).lt(sanitize(value)); + final RangeQueryBuilder termQuery = + QueryBuilders.rangeQuery( getFieldNameForType( value ) ).lt(sanitize(value)); queryBuilders.push( NoOpQueryBuilder.INSTANCE ); @@ -346,8 +339,8 @@ public class EsQueryVistor implements QueryVisitor { final Object value = op.getLiteral().getValue(); - final RangeFilterBuilder termQuery = - FilterBuilders.rangeFilter( getFieldNameForType( value ) ).lte(sanitize(value)); + final RangeQueryBuilder termQuery = + QueryBuilders.rangeQuery( getFieldNameForType( value ) ).lte(sanitize(value)); queryBuilders.push( NoOpQueryBuilder.INSTANCE ); @@ -383,8 +376,8 @@ public class EsQueryVistor implements QueryVisitor { } //it's an exact match, use a filter - final TermFilterBuilder termFilter = - FilterBuilders.termFilter( IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, stringValue ); + final TermQueryBuilder termFilter = + QueryBuilders.termQuery( IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, stringValue ); queryBuilders.push( NoOpQueryBuilder.INSTANCE ); filterBuilders.push( fieldNameTerm( name, termFilter ) ); @@ -394,8 +387,8 @@ public class EsQueryVistor implements QueryVisitor { // assume all other types need prefix - final TermFilterBuilder termQuery = - FilterBuilders.termFilter(getFieldNameForType(value), sanitize(value)); + final TermQueryBuilder termQuery = + QueryBuilders.termQuery(getFieldNameForType(value), sanitize(value)); filterBuilders.push( fieldNameTerm( name, termQuery ) ); @@ -409,8 +402,8 @@ public class EsQueryVistor implements QueryVisitor { final Object value = op.getLiteral().getValue(); - final RangeFilterBuilder rangeQuery = - FilterBuilders.rangeFilter( getFieldNameForType( value ) ).gt(sanitize(value)); + final RangeQueryBuilder rangeQuery = + QueryBuilders.rangeQuery( getFieldNameForType( value ) ).gt(sanitize(value)); filterBuilders.push( fieldNameTerm( name, rangeQuery ) ); @@ -424,8 +417,8 @@ public class EsQueryVistor implements QueryVisitor { Object value = op.getLiteral().getValue(); - final RangeFilterBuilder rangeQuery = - FilterBuilders.rangeFilter( getFieldNameForType( value ) ).gte(sanitize(value)); + final RangeQueryBuilder rangeQuery = + QueryBuilders.rangeQuery( getFieldNameForType( value ) ).gte(sanitize(value)); filterBuilders.push(fieldNameTerm(name, rangeQuery)); @@ -434,12 +427,12 @@ public class EsQueryVistor implements QueryVisitor { @Override - public Optional<FilterBuilder> getFilterBuilder() { + public Optional<QueryBuilder> getFilterBuilder() { if ( filterBuilders.empty() ) { return Optional.absent(); } - final FilterBuilder builder = filterBuilders.peek(); + final QueryBuilder builder = filterBuilders.peek(); if ( !use( builder ) ) { return Optional.absent(); @@ -484,26 +477,26 @@ public class EsQueryVistor implements QueryVisitor { booleanQuery.must( fieldValueQuery ); - return QueryBuilders.nestedQuery(IndexingUtils.ENTITY_FIELDS, booleanQuery); + return QueryBuilders.nestedQuery(IndexingUtils.ENTITY_FIELDS, booleanQuery, ScoreMode.None); } /** * Generate the field name term for the field name for filters - */ - private NestedFilterBuilder fieldNameTerm( final String fieldName, final FilterBuilder fieldValueBuilder ) { + * + private NestedQueryBuilder fieldNameTerm( final String fieldName, final QueryBuilder fieldValueBuilder ) { - final BoolFilterBuilder booleanQuery = FilterBuilders.boolFilter(); + final BoolQueryBuilder booleanQuery = QueryBuilders.boolQuery(); - booleanQuery.must( FilterBuilders.termFilter( IndexingUtils.FIELD_NAME_NESTED, fieldName ) ); + booleanQuery.must( QueryBuilders.termQuery( IndexingUtils.FIELD_NAME_NESTED, fieldName ) ); booleanQuery.must( fieldValueBuilder ); - return FilterBuilders.nestedFilter( IndexingUtils.ENTITY_FIELDS, booleanQuery ); + return QueryBuilders.nestedQuery( IndexingUtils.ENTITY_FIELDS, booleanQuery ); } - +*/ /** * Get the field name for the primitive type */ @@ -578,14 +571,7 @@ public class EsQueryVistor implements QueryVisitor { * Return false if our element is a no-op, true otherwise */ private boolean use( final QueryBuilder queryBuilder ) { - return queryBuilder != NoOpQueryBuilder.INSTANCE; + return queryBuilder != NoOpQueryBuilder.INSTANCE && queryBuilder != NoOpFilterBuilder.INSTANCE; } - - /** - * Return false if our element is a no-op, true otherwise - */ - private boolean use( final FilterBuilder filterBuilder ) { - return filterBuilder != NoOpFilterBuilder.INSTANCE; - } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java index 9eca433..780ecf3 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java @@ -20,22 +20,11 @@ package org.apache.usergrid.persistence.index.impl; -import java.io.IOException; - -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.BaseFilterBuilder; - - /** * Marker interface that does nothing */ -public class NoOpFilterBuilder extends BaseFilterBuilder { +public class NoOpFilterBuilder extends NoOpQueryBuilder { public static final NoOpFilterBuilder INSTANCE = new NoOpFilterBuilder(); - - @Override - protected void doXContent( final XContentBuilder builder, final Params params ) throws IOException { - //no op - } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java index 2498812..69b39ee 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java @@ -22,20 +22,49 @@ package org.apache.usergrid.persistence.index.impl; import java.io.IOException; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.BaseQueryBuilder; +import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryShardContext; /** * Marker interface that does nothing */ -public class NoOpQueryBuilder extends BaseQueryBuilder implements QueryBuilder { +public class NoOpQueryBuilder extends AbstractQueryBuilder implements QueryBuilder { public static final NoOpQueryBuilder INSTANCE = new NoOpQueryBuilder(); @Override + protected void doWriteTo(StreamOutput out) throws IOException { + + } + + @Override protected void doXContent( final XContentBuilder builder, final Params params ) throws IOException { //no op } + + @Override + protected Query doToQuery(QueryShardContext context) throws IOException { + return null; + } + + @Override + protected boolean doEquals(AbstractQueryBuilder other) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + + + @Override + public String getWriteableName() { + return null; + } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java index 5812c6f..3119b4a 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java @@ -23,11 +23,10 @@ package org.apache.usergrid.persistence.index.impl; import org.apache.usergrid.persistence.index.IndexAlias; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.index.query.BoolFilterBuilder; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.TermFilterBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.GeoDistanceSortBuilder; import org.elasticsearch.search.sort.SortBuilders; @@ -50,7 +49,7 @@ import com.google.common.base.Preconditions; import java.util.List; import java.util.Map; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*; import static org.apache.usergrid.persistence.index.impl.SortBuilder.sortPropertyTermFilter; @@ -145,6 +144,7 @@ public class SearchRequestBuilderStrategy { //sort by the entity id if our times are equal //srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order( SortOrder.ASC ) ); + return; } @@ -205,7 +205,7 @@ public class SearchRequestBuilderStrategy { * Create our filter builder. We need to restrict our results on edge search, as well as on types, and any filters * that came from the grammar. */ - private FilterBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor visitor, + private QueryBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor visitor, final SearchTypes searchTypes ) { String context = createContextName( applicationScope, searchEdge ); @@ -218,10 +218,10 @@ public class SearchRequestBuilderStrategy { // Do we need to put the context term first for performance? //make sure we have entity in the context - BoolFilterBuilder boolQueryFilter = FilterBuilders.boolFilter(); + BoolQueryBuilder boolQueryFilter = QueryBuilders.boolQuery(); //add our edge search - boolQueryFilter.must( FilterBuilders.termFilter( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) ); + boolQueryFilter.must( QueryBuilders.termQuery( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) ); /** @@ -231,19 +231,23 @@ public class SearchRequestBuilderStrategy { if ( sourceTypes.length > 0 ) { - final FilterBuilder[] typeTerms = new FilterBuilder[sourceTypes.length]; + final QueryBuilder[] typeTerms = new QueryBuilder[sourceTypes.length]; + + BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); for ( int i = 0; i < sourceTypes.length; i++ ) { - typeTerms[i] = FilterBuilders.termFilter( IndexingUtils.ENTITY_TYPE_FIELDNAME, sourceTypes[i] ); + typeTerms[i] = QueryBuilders.termQuery( IndexingUtils.ENTITY_TYPE_FIELDNAME, sourceTypes[i] ); + boolQuery.filter(typeTerms[i]); } //add all our types, 1 type must match per query - boolQueryFilter.must( FilterBuilders.orFilter( typeTerms ) ); + //boolQueryFilter.must( QueryBuilders.orFilter( typeTerms ) ); + boolQueryFilter.must( boolQuery ); } //if we have a filter from our visitor, add it - Optional<FilterBuilder> queryBuilder = visitor.getFilterBuilder(); + Optional<QueryBuilder> queryBuilder = visitor.getFilterBuilder(); if ( queryBuilder.isPresent() ) { boolQueryFilter.must( queryBuilder.get() ); @@ -263,9 +267,13 @@ public class SearchRequestBuilderStrategy { private FieldSortBuilder createSort( final SortOrder sortOrder, final String fieldName, final String propertyName ) { - final TermFilterBuilder propertyFilter = sortPropertyTermFilter( propertyName ); + final TermQueryBuilder propertyFilter = QueryBuilders.termQuery( ENTITY_FIELDS + "." + IndexingUtils.FIELD_NAME, propertyName ); + FieldSortBuilder fieldSortBuilder = SortBuilders.fieldSort( fieldName ) + .order( sortOrder ) + .setNestedFilter( propertyFilter ) + .setNestedPath(ENTITY_FIELDS ); - return SortBuilders.fieldSort( fieldName ).order( sortOrder ).setNestedFilter( propertyFilter ); + return fieldSortBuilder; } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java index ad78be7..f487747 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java @@ -24,8 +24,8 @@ package org.apache.usergrid.persistence.index.impl; -import org.elasticsearch.index.query.FilterBuilders; -import org.elasticsearch.index.query.TermFilterBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; /** @@ -37,8 +37,8 @@ public class SortBuilder { /** * Create a term filter for our sorts */ - public static TermFilterBuilder sortPropertyTermFilter( final String propertyName ) { - return FilterBuilders.termFilter( IndexingUtils.FIELD_NAME, propertyName ); + public static TermQueryBuilder sortPropertyTermFilter( final String propertyName ) { + return QueryBuilders.termQuery( IndexingUtils.FIELD_NAME, propertyName ); } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java index 273f23f..5e2d50f 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java @@ -19,19 +19,13 @@ package org.apache.usergrid.persistence.index.query.tree; -import java.util.Collection; -import java.util.Map; -import java.util.Set; - import org.apache.usergrid.persistence.index.exceptions.NoFullTextIndexException; import org.apache.usergrid.persistence.index.exceptions.NoIndexException; import org.apache.usergrid.persistence.index.exceptions.IndexException; import org.apache.usergrid.persistence.index.impl.GeoSortFields; -import org.apache.usergrid.persistence.index.query.SortPredicate; -import org.elasticsearch.index.query.FilterBuilder; + import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.sort.GeoDistanceSortBuilder; import com.google.common.base.Optional; @@ -108,7 +102,7 @@ public interface QueryVisitor { * Return any filters created during parsing * @return */ - Optional<FilterBuilder> getFilterBuilder(); + Optional<QueryBuilder> getFilterBuilder(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb593d46/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json index 3d2517c..5142d76 100644 --- a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json +++ b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json @@ -73,8 +73,6 @@ }, "location": { "type": "geo_point", - "lat_lon": true, - "geohash": true, "doc_values": true }, "string": {
