[USERGRID-1336] Initial commit of a simple query analyzer feature.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a4941462 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a4941462 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a4941462 Branch: refs/heads/query-analyzer Commit: a494146250ea92c6d0ebbb9c579ff194525f5eed Parents: f8e644d Author: Michael Russo <russomich...@google.com> Authored: Fri Feb 24 14:49:09 2017 -0800 Committer: Michael Russo <russomich...@google.com> Committed: Fri Feb 24 14:49:09 2017 -0800 ---------------------------------------------------------------------- .../corepersistence/CpRelationManager.java | 9 ++ .../pipeline/builder/IdBuilder.java | 10 +- .../pipeline/read/FilterFactory.java | 7 +- .../search/AbstractElasticSearchFilter.java | 21 ++- .../read/search/SearchCollectionFilter.java | 5 +- .../read/search/SearchConnectionFilter.java | 5 +- .../service/AggregationServiceImpl.java | 6 +- .../service/CollectionSearch.java | 10 ++ .../service/CollectionServiceImpl.java | 2 +- .../service/ConnectionSearch.java | 12 ++ .../service/ConnectionServiceImpl.java | 4 +- .../org/apache/usergrid/persistence/Query.java | 9 ++ .../org/apache/usergrid/utils/JsonUtils.java | 2 +- .../corepersistence/StaleIndexCleanupTest.java | 2 +- .../index/AsyncIndexServiceTest.java | 2 +- .../corepersistence/index/IndexServiceTest.java | 8 +- .../usergrid/persistence/index/EntityIndex.java | 11 +- .../usergrid/persistence/index/IndexFig.java | 39 ++++++ .../persistence/index/QueryAnalyzer.java | 129 +++++++++++++++++++ .../QueryAnalyzerEnforcementException.java | 45 +++++++ .../exceptions/QueryAnalyzerException.java | 57 ++++++++ .../index/impl/EsEntityIndexImpl.java | 61 ++++++++- .../persistence/index/impl/EntityIndexTest.java | 86 ++++++------- .../persistence/index/impl/GeoPagingTest.java | 2 +- .../index/impl/IndexLoadTestsIT.java | 2 +- .../rest/applications/CollectionResource.java | 17 --- .../rest/applications/ServiceResource.java | 9 +- .../exceptions/AbstractExceptionMapper.java | 17 ++- ...QueryAnalyzerEnforcementExceptionMapper.java | 51 ++++++++ .../QueryAnalyzerExceptionMapper.java | 69 ++++++++++ .../usergrid/services/AbstractService.java | 2 + .../usergrid/services/ServiceManager.java | 15 +-- .../usergrid/services/ServiceRequest.java | 26 ++-- 33 files changed, 626 insertions(+), 126 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index 4a759ac..c02ca7d 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@ -629,6 +629,7 @@ public class CpRelationManager implements RelationManager { final Query toExecute = adjustQuery( query ); final Optional<String> queryString = query.isGraphSearch()? Optional.<String>absent(): query.getQl(); final Id ownerId = headEntity.asId(); + final boolean analyzeOnly = query.getAnalyzeOnly(); if(query.getLevel() == Level.IDS ){ @@ -642,6 +643,8 @@ public class CpRelationManager implements RelationManager { new CollectionSearch( applicationScope, ownerId, collectionName, collection.getType(), toExecute.getLimit(), queryString, cursor ); + search.setAnalyzeOnly(analyzeOnly); + return collectionService.searchCollectionIds( search ); } }.next(); @@ -658,6 +661,8 @@ public class CpRelationManager implements RelationManager { new CollectionSearch( applicationScope, ownerId, collectionName, collection.getType(), toExecute.getLimit(), queryString, cursor ); + search.setAnalyzeOnly(analyzeOnly); + return collectionService.searchCollection( search ); } }.next(); @@ -919,6 +924,8 @@ public class CpRelationManager implements RelationManager { headEntity = em.validate( headEntity ); + final boolean analyzeOnly = query.getAnalyzeOnly(); + final Query toExecute = adjustQuery( query ); @@ -951,6 +958,7 @@ public class CpRelationManager implements RelationManager { final ConnectionSearch search = new ConnectionSearch( applicationScope, sourceId, entityType, connection, toExecute.getLimit(), queryString, cursor, isConnecting ); + search.setAnalyzeOnly(analyzeOnly); return connectionService.searchConnectionAsRefs( search ); } }.next(); @@ -966,6 +974,7 @@ public class CpRelationManager implements RelationManager { final ConnectionSearch search = new ConnectionSearch( applicationScope, sourceId, entityType, connection, toExecute.getLimit(), queryString, cursor, isConnecting ); + search.setAnalyzeOnly(analyzeOnly); return connectionService.searchConnection( search ); } }.next(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java index 85e9069..a7f9ad9 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java @@ -117,12 +117,13 @@ public class IdBuilder { * @param collectionName The name of the collection * @param ql The user's query to execute * @param entityType The type of the entity + * @param analyzeOnly * @return Candidate results */ - public CandidateBuilder searchCollection( final String collectionName, final String ql, final String entityType ) { + public CandidateBuilder searchCollection(final String collectionName, final String ql, final String entityType, boolean analyzeOnly) { final Pipeline<FilterResult<Candidate>> newFilter = pipeline.withFilter( filterFactory.searchCollectionFilter( - ql, collectionName, entityType ) ); + ql, collectionName, entityType, analyzeOnly ) ); return new CandidateBuilder( newFilter, filterFactory ); } @@ -133,13 +134,14 @@ public class IdBuilder { * @param connectionName The connection name to search * @param ql The query to execute * @param entityType The optional type of entity. If this is absent, all entity types in the connection will be searched + * @param analyzeOnly * @return Candidate results */ - public CandidateBuilder searchConnection( final String connectionName, final String ql , final Optional<String> entityType) { + public CandidateBuilder searchConnection(final String connectionName, final String ql, final Optional<String> entityType, boolean analyzeOnly) { final Pipeline<FilterResult<Candidate>> newFilter = pipeline.withFilter( filterFactory.searchConnectionFilter( - ql, connectionName, entityType ) ); + ql, connectionName, entityType,analyzeOnly ) ); return new CandidateBuilder( newFilter, filterFactory ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java index 4b615d8..7b61b3d 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java @@ -96,7 +96,8 @@ public interface FilterFactory { */ SearchCollectionFilter searchCollectionFilter( @Assisted( "query" ) final String query, @Assisted( "collectionName" ) final String collectionName, - @Assisted( "entityType" ) final String entityType ); + @Assisted( "entityType" ) final String entityType, + @Assisted( "analyzeOnly") final boolean analyzeOnly); /** @@ -108,8 +109,8 @@ public interface FilterFactory { */ SearchConnectionFilter searchConnectionFilter( @Assisted( "query" ) final String query, @Assisted( "connectionName" ) final String connectionName, - @Assisted( "connectedEntityType" ) - final Optional<String> connectedEntityType ); + @Assisted( "connectedEntityType" ) final Optional<String> connectedEntityType, + @Assisted( "analyzeOnly") final boolean analyzeOnly); /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java index 4bf723e..9f6be2a 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java @@ -23,6 +23,8 @@ package org.apache.usergrid.corepersistence.pipeline.read.search; import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory; import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.index.*; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerEnforcementException; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,19 +56,21 @@ public abstract class AbstractElasticSearchFilter extends AbstractPathFilter<Id, private final IndexLocationStrategyFactory indexLocationStrategyFactory; private final String query; private final Timer searchTimer; + private final boolean analyzeOnly; /** * Create a new instance of our command */ - public AbstractElasticSearchFilter( final EntityIndexFactory entityIndexFactory, - final MetricsFactory metricsFactory, - final IndexLocationStrategyFactory indexLocationStrategyFactory, - final String query ) { + public AbstractElasticSearchFilter(final EntityIndexFactory entityIndexFactory, + final MetricsFactory metricsFactory, + final IndexLocationStrategyFactory indexLocationStrategyFactory, + final String query, boolean analyzeOnly) { this.entityIndexFactory = entityIndexFactory; this.indexLocationStrategyFactory = indexLocationStrategyFactory; this.query = query; this.searchTimer = metricsFactory.getTimer( AbstractElasticSearchFilter.class, "query.search" ); + this.analyzeOnly = analyzeOnly; } @@ -123,7 +127,7 @@ public abstract class AbstractElasticSearchFilter extends AbstractPathFilter<Id, try { final CandidateResults candidateResults = - applicationEntityIndex.search( searchEdge, searchTypes, query, limit, currentOffSet, propertiesWithType ); + applicationEntityIndex.search( searchEdge, searchTypes, query, limit, currentOffSet, propertiesWithType, analyzeOnly ); Collection<SelectFieldMapping> fieldMappingCollection = candidateResults.getGetFieldMappings(); @@ -156,8 +160,11 @@ public abstract class AbstractElasticSearchFilter extends AbstractPathFilter<Id, } catch ( Throwable t ) { - - logger.error( "Unable to search candidates", t ); + // query analyzer exceptions are short circuits initiated by an exception, but is not really an error + // still rethrow because it's mapped later + if (!(t instanceof QueryAnalyzerException || t instanceof QueryAnalyzerEnforcementException) ){ + logger.error( "Unable to search candidates", t ); + } subscriber.onError( t ); } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java index 4fc6179..214bcbb 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java @@ -52,8 +52,9 @@ public class SearchCollectionFilter extends AbstractElasticSearchFilter { final MetricsFactory metricsFactory, @Assisted( "query" ) final String query, @Assisted( "collectionName" ) final String collectionName, - @Assisted( "entityType" ) final String entityType ) { - super( entityIndexFactory, metricsFactory, indexLocationStrategyFactory, query ); + @Assisted( "entityType" ) final String entityType, + @Assisted( "analyzeOnly") final boolean analyzeOnly) { + super( entityIndexFactory, metricsFactory, indexLocationStrategyFactory, query, analyzeOnly ); this.collectionName = collectionName; this.entityType = entityType; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java index d484ade..4a61120 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java @@ -50,8 +50,9 @@ public class SearchConnectionFilter extends AbstractElasticSearchFilter { final IndexLocationStrategyFactory indexLocationStrategyFactory, @Assisted( "query" ) final String query, @Assisted( "connectionName" ) final String connectionName, - @Assisted( "connectedEntityType" ) final Optional<String> connectedEntityType ) { - super( entityIndexFactory, metricsFactory, indexLocationStrategyFactory, query ); + @Assisted( "connectedEntityType" ) final Optional<String> connectedEntityType, + @Assisted( "analyzeOnly") final boolean analyzeOnly) { + super( entityIndexFactory, metricsFactory, indexLocationStrategyFactory, query, analyzeOnly); this.connectionName = connectionName; this.connectedEntityType = connectedEntityType; http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java index 44010db..6588486 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java @@ -73,7 +73,7 @@ public class AggregationServiceImpl implements AggregationService { MathObservable.sumLong( graphManager.getEdgeTypesFromSource(new SimpleSearchEdgeType(applicationScope.getApplication(), CpNamingUtils.EDGE_COLL_PREFIX, Optional.<String>absent())) .map(type -> CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type)) - .map(edge -> entityIndex.getEntitySize(edge)) + .map(edge -> entityIndex.getTotalEntitySizeInBytes(edge)) ), sumTimer).toBlocking().last(); return sum.longValue(); @@ -90,7 +90,7 @@ public class AggregationServiceImpl implements AggregationService { { SearchEdge edge = CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type); final String collectionName = CpNamingUtils.getCollectionNameFromEdgeName(type); - long sumType = entityIndex.getEntitySize(edge); + long sumType = entityIndex.getTotalEntitySizeInBytes(edge); map.put(collectionName,sumType); }) ) @@ -103,7 +103,7 @@ public class AggregationServiceImpl implements AggregationService { public long getSize(ApplicationScope applicationScope, SearchEdge edge) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); - return entityIndex.getEntitySize(edge); + return entityIndex.getTotalEntitySizeInBytes(edge); } @Override http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java index 602a5b6..6240028 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java @@ -41,6 +41,7 @@ public class CollectionSearch { private final Optional<String> query; private final Optional<String> cursor; private Level level = Level.ALL; + private boolean analyzeOnly; public CollectionSearch( final ApplicationScope applicationScope, final Id collectionOwnerId, final String @@ -53,6 +54,7 @@ public class CollectionSearch { this.limit = limit; this.query = query; this.cursor = cursor; + this.analyzeOnly = false; } @@ -93,4 +95,12 @@ public class CollectionSearch { public void setResultsLevel(Level level){ this.level = level; } public Level getResultsLevel(){ return level; } + + public boolean getAnalyzeOnly() { + return analyzeOnly; + } + + public void setAnalyzeOnly(final boolean analyzeOnly){ + this.analyzeOnly = analyzeOnly; + } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java index 9244315..7684050 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java @@ -68,7 +68,7 @@ public class CollectionServiceImpl implements CollectionService { results = pipelineBuilder.traverseCollection( collectionName ).loadEntities(); } else { - results = pipelineBuilder.searchCollection( collectionName, query.get(),search.getEntityType()).loadEntities(); + results = pipelineBuilder.searchCollection( collectionName, query.get(),search.getEntityType(), search.getAnalyzeOnly()).loadEntities(); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java index 8ad57fb..98f36be 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java @@ -37,6 +37,8 @@ public class ConnectionSearch { private final Optional<String> query; private final Optional<String> cursor; private final boolean isConnecting; + private boolean analyzeOnly; + public ConnectionSearch( final ApplicationScope applicationScope, final Id sourceNodeId, final Optional<String> entityType, @@ -50,6 +52,8 @@ public class ConnectionSearch { this.query = query; this.cursor = cursor; this.isConnecting = isConnecting; + this.analyzeOnly = false; + } @@ -90,4 +94,12 @@ public class ConnectionSearch { public boolean getIsConnecting(){ return isConnecting; } + + public boolean getAnalyzeOnly() { + return analyzeOnly; + } + + public void setAnalyzeOnly(final boolean analyzeOnly){ + this.analyzeOnly = analyzeOnly; + } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java index 926c676..7c70a3d 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java @@ -106,7 +106,7 @@ public class ConnectionServiceImpl implements ConnectionService { else { results = - pipelineBuilder.searchConnection( search.getConnectionName(), query.get(), search.getEntityType() ) + pipelineBuilder.searchConnection( search.getConnectionName(), query.get(), search.getEntityType(), search.getAnalyzeOnly() ) .loadEntities(); } @@ -135,7 +135,7 @@ public class ConnectionServiceImpl implements ConnectionService { } else { traversedIds = - pipelineBuilder.searchConnection( connectionName, query.get(), search.getEntityType() ).loadIds(); + pipelineBuilder.searchConnection( connectionName, query.get(), search.getEntityType(), search.getAnalyzeOnly() ).loadIds(); } //create connection refs http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java index d68c085..1015f76 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java @@ -78,6 +78,7 @@ public class Query { private String collection; private String ql; private Collection<SelectFieldMapping> selectFields; + private boolean analyzeOnly = false; private static ObjectMapper mapper = new ObjectMapper(); @@ -120,6 +121,7 @@ public class Query { ? new ArrayList<>( q.counterFilters ) : null; collection = q.collection; level = q.level; + analyzeOnly = q.analyzeOnly; } @@ -481,6 +483,13 @@ public class Query { this.permissions = permissions; } + public void setAnalyzeOnly(final boolean analyzeOnly){ + this.analyzeOnly = analyzeOnly; + } + + public boolean getAnalyzeOnly(){ + return analyzeOnly; + } public boolean isMergeSelectResults() { return mergeSelectResults; http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java b/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java index 3bb96f9..6c1875b 100644 --- a/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java +++ b/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java @@ -83,7 +83,7 @@ public class JsonUtils { /** Converts object to JSON string, throws runtime exception JsonWriteException on failure. */ public static String mapToFormattedJsonString( Object obj ) { try { - return indentObjectMapper.writeValueAsString( obj ); + return indentObjectMapper.writerWithDefaultPrettyPrinter().writeValueAsString( obj ); } catch ( Throwable t ) { if (logger.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java index 65c373f..abe2615 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java @@ -344,7 +344,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT { SearchEdge is = CpNamingUtils.createCollectionSearchEdge( rootId, collName ); - return ei.search( is, SearchTypes.fromTypes( type ), query, 1000, 0 ); + return ei.search( is, SearchTypes.fromTypes( type ), query, 1000, 0, false ); } /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java index 366ab34..cecc3b2 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java @@ -182,7 +182,7 @@ public abstract class AsyncIndexServiceTest { for ( int i = 0; i < attempts; i++ ) { final CandidateResults candidateResults = - entityIndex.search( searchEdge, searchTypes, "select *", 100, 0 ); + entityIndex.search( searchEdge, searchTypes, "select *", 100, 0, false ); if ( candidateResults.size() == expectedSize ) { return candidateResults; http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java index ecc2b46..f47afe6 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java @@ -353,7 +353,7 @@ public class IndexServiceTest { //ensure that no edges remain CandidateResults connectionResultsEmpty = EntityIndex.search( connectionSearchEdge, - SearchTypes.fromTypes( "thing" ),"select *",10,0 ); + SearchTypes.fromTypes( "thing" ),"select *",10,0, false ); assertEquals(1,connectionResultsEmpty.size()); @@ -375,7 +375,7 @@ public class IndexServiceTest { //ensure that no edges remain connectionResultsEmpty = EntityIndex.search( connectionSearchEdge, - SearchTypes.fromTypes( "thing" ),"select *",10,0 ); + SearchTypes.fromTypes( "thing" ),"select *",10,0, false ); assertEquals(0,connectionResultsEmpty.size()); @@ -434,7 +434,7 @@ public class IndexServiceTest { //ensure that no edges remain final CandidateResults connectionResultsEmpty = EntityIndex.search( connectionSearchEdge, - SearchTypes.fromTypes( "things" ),"select *",10,0 ); + SearchTypes.fromTypes( "things" ),"select *",10,0, false ); assertEquals(0,connectionResultsEmpty.size()); } @@ -535,7 +535,7 @@ public class IndexServiceTest { String ql = "select *"; for ( int i = 0; i < attempts; i++ ) { final CandidateResults candidateResults = - EntityIndex.search( searchEdge, searchTypes, ql , 100, 0 ); + EntityIndex.search( searchEdge, searchTypes, ql , 100, 0, false ); if ( candidateResults.size() == expectedSize ) { return candidateResults; http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index ee12ec8..14020a9 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@ -76,7 +76,7 @@ public interface EntityIndex extends CPManager { * @param edge * @return */ - long getEntitySize(final SearchEdge edge); + long getTotalEntitySizeInBytes(final SearchEdge edge); /** * Initialize the index if necessary. This is an idempotent operation and should not create an index @@ -98,10 +98,12 @@ public interface EntityIndex extends CPManager { * @param query The query to execute * @param limit The limit of values to return * @param offset The offset to query on + * @param analyzeOnly This optional param will instruct the query processing to only analyze the query and + * provide info but not actually execute the query. * @return */ CandidateResults search(final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, - final int limit, final int offset); + final int limit, final int offset, final boolean analyzeOnly); /** * Search on every document in the specified search edge. Also search by the types if specified @@ -113,10 +115,13 @@ public interface EntityIndex extends CPManager { * @param offset The offset to query on * @param fieldsWithType An optional param that allows the caller to provide schema related info which might * relate to data in the query, such as sort predicate types + * @param analyzeOnly This optional param will instruct the query processing to only analyze the query and + * provide info but not actually execute the query. * @return */ CandidateResults search(final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, - final int limit, final int offset, final Map<String, Class> fieldsWithType); + final int limit, final int offset, final Map<String, Class> fieldsWithType, + final boolean analyzeOnly); /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java index c4596a2..e23961c 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java @@ -62,6 +62,19 @@ public interface IndexFig extends GuicyFig { String ELASTICSEARCH_VERSION_QUERY_LIMIT = "elasticsearch.version_query_limit"; + String USERGRID_QUERYANALYZER_OPERAND_COUNT = "usergrid.queryanalyzer.operand_count"; + + String USERGRID_QUERYANALYZER_SORTPREDICATE_COUNT = "usergrid.queryanalyzer.sortpredicate_count"; + + String USERGRID_QUERYANALYZER_COLLECTIONSIZE = "usergrid.queryanalyzer.collectionsize_bytes"; + + String USERGRID_QUERYANALYZER_INDEXSIZE = "usergrid.queryanalyzer.indexsize_bytes"; + + String USERGRID_QUERYANALYZER_ENFORCE = "usergrid.queryanalyzer.enforce"; + + + + /** * Comma-separated list of Elasticsearch hosts. @@ -196,7 +209,33 @@ public interface IndexFig extends GuicyFig { @Key( "elasticsearch_queue_error_sleep_ms" ) long getSleepTimeForQueueError(); + @Default("100") @Key( ELASTICSEARCH_VERSION_QUERY_LIMIT ) int getVersionQueryLimit(); + + + @Default("8") + @Key( USERGRID_QUERYANALYZER_OPERAND_COUNT ) + int getQueryBreakerErrorOperandCount(); + + + @Default("8") + @Key( USERGRID_QUERYANALYZER_SORTPREDICATE_COUNT ) + int getQueryBreakerErrorSortPredicateCount(); + + + @Default("500000000L") // 500 MB + @Key(USERGRID_QUERYANALYZER_COLLECTIONSIZE) + long getQueryBreakerErrorCollectionSizeBytes(); + + + @Default("10000000000") // 10 GB + @Key( USERGRID_QUERYANALYZER_INDEXSIZE ) + long getQueryBreakerErrorIndexSizeBytes(); + + + @Default("true") + @Key( USERGRID_QUERYANALYZER_ENFORCE ) + boolean enforceQueryBreaker(); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java new file mode 100644 index 0000000..86f1e5d --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. The ASF licenses this file to You + * under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. For additional information regarding + * copyright in this work, please see the NOTICE file in the top level + * directory of this distribution. + */ +package org.apache.usergrid.persistence.index; + +import org.apache.usergrid.persistence.index.query.ParsedQuery; +import org.apache.usergrid.persistence.index.query.tree.Operand; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class QueryAnalyzer { + + public static final String v_predicate_count = "sort_predicate_count_exceeded"; + public static final String v_operand_count = "operand_count_exceeded"; + public static final String v_large_collection = "large_collection_size_bytes"; + public static final String v_large_index = "large_index_size_bytes"; + + public static final String k_violation = "violation"; + public static final String k_limit = "limit"; + public static final String k_actual = "actual"; + + + public static List<Map<String, Object>> analyze(final ParsedQuery parsedQuery, final long collectionSizeInBytes, + final long indexSizeInBytes, final IndexFig indexFig ) { + + List<Map<String, Object>> violations = new ArrayList<>(); + + // get configured breaker values + final int errorPredicateCount = indexFig.getQueryBreakerErrorSortPredicateCount(); + final int errorOperandCount = indexFig.getQueryBreakerErrorOperandCount(); + final long errorCollectionSizeBytes = indexFig.getQueryBreakerErrorCollectionSizeBytes(); + final long errorIndexSizeBytes = indexFig.getQueryBreakerErrorIndexSizeBytes(); + + + // get the actual values to compare against the configured enforcement values + int queryPredicatesSize = parsedQuery.getSortPredicates().size(); + int queryOperandCount = getTotalChildCount(parsedQuery.getRootOperand()); + + // large indexes can cause issues, this is never returned from the API and only logged + if( indexSizeInBytes > errorIndexSizeBytes ){ + violations.add(new HashMap<String, Object>(3){{ + put(k_violation, v_large_index); + put(k_limit, errorIndexSizeBytes); + put(k_actual, indexSizeInBytes); + }}); + } + + // large collections mean that sorts and other complex queries can impact the query service (Elasticsearch) + if (collectionSizeInBytes > errorCollectionSizeBytes ){ + violations.add(new HashMap<String, Object>(3){{ + put(k_violation, v_large_collection); + put(k_limit, errorCollectionSizeBytes); + put(k_actual, collectionSizeInBytes); + }}); + } + + // complex queries can be determined from the # of operands and sort predicates + if ( queryPredicatesSize > errorPredicateCount){ + violations.add(new HashMap<String, Object>(3){{ + put(k_violation, v_predicate_count); + put(k_limit, errorPredicateCount); + put(k_actual, queryPredicatesSize); + }}); + } + if (queryOperandCount > errorOperandCount){ + violations.add(new HashMap<String, Object>(3){{ + put(k_violation, v_operand_count); + put(k_limit, errorOperandCount); + put(k_actual, queryOperandCount); + }}); + } + + return violations; + + } + + public static String violationsAsString(List<Map<String, Object>> violations, String originalQuery){ + + final StringBuilder logMessage = new StringBuilder(); + logMessage.append( "QueryAnalyzer Violations Detected [").append(violations.size()).append("]: [" ); + violations.forEach(violation -> { + + final StringBuilder violationMessage = new StringBuilder(); + violation.forEach((k,v) -> { + violationMessage.append(k).append(":").append(v).append(","); + + }); + violationMessage.deleteCharAt(violationMessage.length()-1); + logMessage.append(" (").append(violationMessage).append(") "); + }); + logMessage.append("]"); + logMessage.append(" [Original Query: ").append(originalQuery).append("]"); + return logMessage.toString(); + + } + + private static int getTotalChildCount(Operand rootOperand){ + int count = 0; + if( rootOperand != null) { + count ++; + if (rootOperand.getChildren() != null) { + for (Object child : rootOperand.getChildren()) { + if (child instanceof Operand) { + count += getTotalChildCount((Operand) child); + } + } + } + } + return count; + } + +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerEnforcementException.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerEnforcementException.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerEnforcementException.java new file mode 100644 index 0000000..454fbe1 --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerEnforcementException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. The ASF licenses this file to You + * under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. For additional information regarding + * copyright in this work, please see the NOTICE file in the top level + * directory of this distribution. + */ +package org.apache.usergrid.persistence.index.exceptions; + + +import org.apache.usergrid.persistence.index.QueryAnalyzer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class QueryAnalyzerEnforcementException extends RuntimeException { + + private List<Map<String, Object>> violations = new ArrayList<>(); + private String originalQuery; + + public QueryAnalyzerEnforcementException(final List<Map<String, Object>> violations, final String originalQuery) { + super("Query Analyzer Enforced"); + this.violations = violations; + this.originalQuery = originalQuery; + } + + public List<Map<String, Object>> getViolations(){ + return violations; + } + + public String getErrorMessage() { + return QueryAnalyzer.violationsAsString(violations, originalQuery); + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerException.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerException.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerException.java new file mode 100644 index 0000000..c392822 --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerException.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. The ASF licenses this file to You + * under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. For additional information regarding + * copyright in this work, please see the NOTICE file in the top level + * directory of this distribution. + */ +package org.apache.usergrid.persistence.index.exceptions; + + +import org.apache.usergrid.persistence.index.QueryAnalyzer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public class QueryAnalyzerException extends RuntimeException { + + private List<Map<String, Object>> violations = new ArrayList<>(); + private String originalQuery; + private UUID applicationUUID; + + public QueryAnalyzerException(final List<Map<String, Object>> violations, final String originalQuery, + final UUID applicationUUID) { + super("Query Analyzer"); + this.violations = violations; + this.originalQuery = originalQuery; + this.applicationUUID = applicationUUID; + } + + public List<Map<String, Object>> getViolations(){ + return violations; + } + + public String getErrorMessage() { + return QueryAnalyzer.violationsAsString(violations, originalQuery); + } + + public String getOriginalQuery(){ + return originalQuery; + } + + public UUID getApplicationUUID(){ + return applicationUUID; + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 3615d86..26e8f68 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -21,6 +21,8 @@ package org.apache.usergrid.persistence.index.impl; import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; import com.google.common.base.*; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; import com.google.inject.Inject; @@ -36,6 +38,8 @@ import org.apache.usergrid.persistence.core.util.ValidationUtils; import org.apache.usergrid.persistence.index.*; import org.apache.usergrid.persistence.index.ElasticSearchQueryBuilder.SearchRequestBuilderStrategyV2; import org.apache.usergrid.persistence.index.exceptions.IndexException; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerException; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerEnforcementException; import org.apache.usergrid.persistence.index.migration.IndexDataVersions; import org.apache.usergrid.persistence.index.query.ParsedQuery; import org.apache.usergrid.persistence.index.query.ParsedQueryBuilder; @@ -54,6 +58,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -78,6 +84,7 @@ import rx.Observable; import java.io.IOException; import java.net.URL; import java.util.*; +import java.util.concurrent.TimeUnit; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.applicationId; @@ -129,6 +136,10 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { private Meter refreshIndexMeter; + private Cache<String, Long> sizeCache = + CacheBuilder.newBuilder().maximumSize( 1000 ).expireAfterWrite(5, TimeUnit.MINUTES).build(); + + @Inject public EsEntityIndexImpl( final EsProvider provider, final IndexCache indexCache, @@ -412,12 +423,13 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { } public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, - final int limit, final int offset ) { - return search(searchEdge, searchTypes, query, limit, offset, new HashMap<>(0)); + final int limit, final int offset, final boolean analyzeOnly ) { + return search(searchEdge, searchTypes, query, limit, offset, new HashMap<>(0), analyzeOnly); } public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, - final int limit, final int offset, final Map<String, Class> fieldsWithType ) { + final int limit, final int offset, final Map<String, Class> fieldsWithType, + final boolean analyzeOnly ) { IndexValidationUtils.validateSearchEdge(searchEdge); Preconditions.checkNotNull(searchTypes, "searchTypes cannot be null"); @@ -442,6 +454,36 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { } + final String cacheKey = applicationScope.getApplication().getUuid().toString()+"_"+searchEdge.getEdgeName(); + final Object totalEdgeSizeFromCache = sizeCache.getIfPresent(cacheKey); + long totalEdgeSizeInBytes; + if (totalEdgeSizeFromCache == null){ + totalEdgeSizeInBytes = getTotalEntitySizeInBytes(searchEdge); + sizeCache.put(cacheKey, totalEdgeSizeInBytes); + }else{ + totalEdgeSizeInBytes = (long) totalEdgeSizeFromCache; + } + + final Object totalIndexSizeFromCache = sizeCache.getIfPresent(indexLocationStrategy.getIndexRootName()); + long totalIndexSizeInBytes; + if (totalIndexSizeFromCache == null){ + totalIndexSizeInBytes = getIndexSize(); + sizeCache.put(indexLocationStrategy.getIndexRootName(), totalIndexSizeInBytes); + }else{ + totalIndexSizeInBytes = (long) totalIndexSizeFromCache; + } + + List<Map<String, Object>> violations = QueryAnalyzer.analyze(parsedQuery, totalEdgeSizeInBytes, totalIndexSizeInBytes, indexFig); + if(indexFig.enforceQueryBreaker() && violations.size() > 0){ + throw new QueryAnalyzerEnforcementException(violations, parsedQuery.getOriginalQuery()); + }else{ + logger.warn( QueryAnalyzer.violationsAsString(violations, parsedQuery.getOriginalQuery()) ); + } + + if(analyzeOnly){ + throw new QueryAnalyzerException(violations, parsedQuery.getOriginalQuery(), applicationScope.getApplication().getUuid()); + } + final SearchRequestBuilder srb = searchRequest .getBuilder( searchEdge, searchTypes, visitor, limit, offset, parsedQuery.getSortPredicates(), fieldsWithType ) .setTimeout(TimeValue.timeValueMillis(queryTimeout)); @@ -824,9 +866,20 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { return Health.RED; } + private long getIndexSize(){ + final IndicesStatsResponse statsResponse = esProvider.getClient() + .admin() + .indices() + .prepareStats(indexLocationStrategy.getIndexInitialName()) + .all() + .execute() + .actionGet(); + final CommonStats indexStats = statsResponse.getIndex(indexLocationStrategy.getIndexInitialName()).getTotal(); + return indexStats.getStore().getSizeInBytes(); + } @Override - public long getEntitySize(final SearchEdge edge){ + public long getTotalEntitySizeInBytes(final SearchEdge edge){ //"term":{"edgeName":"zzzcollzzz|roles"} SearchRequestBuilder builder = searchRequestBuilderStrategyV2.getBuilder(); builder.setQuery(new TermQueryBuilder("edgeSearch",IndexingUtils.createContextName(applicationScope,edge))); http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index e64db83..ac7d10d 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -160,7 +160,7 @@ public class EntityIndexTest extends BaseIT { StopWatch timer = new StopWatch(); timer.start(); CandidateResults candidateResults = - entityIndex.search( indexEdge, searchTypes, "select * where testfield = 'test' order by ordinal", 100, 0 ); + entityIndex.search( indexEdge, searchTypes, "select * where testfield = 'test' order by ordinal", 100, 0, false ); timer.stop(); @@ -182,11 +182,11 @@ public class EntityIndexTest extends BaseIT { //make sure we can query uuids out as strings and not wrapped candidateResults = - entityIndex.search( indexEdge, searchTypes, "select * where testuuid = '"+uuid+"'", 100, 0 ); + entityIndex.search( indexEdge, searchTypes, "select * where testuuid = '"+uuid+"'", 100, 0, false ); assertEquals(entity1.getId(),candidateResults.get(0).getId()); candidateResults = - entityIndex.search( indexEdge, searchTypes, "select * where testuuid = "+uuid, 100, 0 ); + entityIndex.search( indexEdge, searchTypes, "select * where testuuid = "+uuid, 100, 0, false); assertEquals(entity1.getId(),candidateResults.get(0).getId()); } @@ -369,7 +369,7 @@ public class EntityIndexTest extends BaseIT { entityIndex.refreshAsync().toBlocking().first(); CandidateResults candidateResults = entityIndex - .search( searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0 ); + .search( searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0, false ); assertEquals( 1, candidateResults.size() ); EntityIndexBatch batch = entityIndex.createBatch(); @@ -378,7 +378,7 @@ public class EntityIndexTest extends BaseIT { entityIndex.refreshAsync().toBlocking().first(); candidateResults = entityIndex - .search(searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0 ); + .search(searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0, false ); assertEquals(0, candidateResults.size()); } @@ -390,7 +390,7 @@ public class EntityIndexTest extends BaseIT { StopWatch timer = new StopWatch(); timer.start(); - CandidateResults candidateResults = entityIndex.search( scope, searchTypes, queryString, 1000, 0 ); + CandidateResults candidateResults = entityIndex.search( scope, searchTypes, queryString, 1000, 0, false ); timer.stop(); @@ -518,7 +518,7 @@ public class EntityIndexTest extends BaseIT { final String query = "where username = 'edanuff'"; - CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0); + CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false); assertEquals( user.getId(), r.get( 0 ).getId()); batch.deindex( indexSCope, user.getId(), user.getVersion() ); @@ -528,7 +528,7 @@ public class EntityIndexTest extends BaseIT { // EntityRef - r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0 ); + r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false ); assertFalse( r.iterator().hasNext() ); } @@ -589,16 +589,16 @@ public class EntityIndexTest extends BaseIT { final SearchTypes searchTypes = SearchTypes.fromTypes( "user" ); - CandidateResults r = entityIndex.search( indexScope, searchTypes, "where username = 'bill'", 10, 0); + CandidateResults r = entityIndex.search( indexScope, searchTypes, "where username = 'bill'", 10, 0, false); assertEquals( bill.getId(), r.get( 0 ).getId() ); - r = entityIndex.search( indexScope, searchTypes, "where username = 'fred'", 10, 0); + r = entityIndex.search( indexScope, searchTypes, "where username = 'fred'", 10, 0, false); assertEquals(fred.getId(), r.get(0).getId()); - r = entityIndex.search( indexScope, searchTypes, "where age = 41", 10, 0); + r = entityIndex.search( indexScope, searchTypes, "where age = 41", 10, 0, false); assertEquals(fred.getId(), r.get(0).getId()); - r = entityIndex.search( indexScope, searchTypes, "where age = 'thirtysomething'", 10, 0); + r = entityIndex.search( indexScope, searchTypes, "where age = 'thirtysomething'", 10, 0, false); assertEquals(bill.getId(), r.get(0).getId()); } @@ -677,8 +677,8 @@ public class EntityIndexTest extends BaseIT { int i ; for ( i=0; i < expectedPages; i++ ) { final CandidateResults results = !offset.isPresent() - ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit, 0 ) - : entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit); + ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit, 0, false ) + : entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit, false); assertEquals(limit, results.size()); @@ -690,7 +690,7 @@ public class EntityIndexTest extends BaseIT { } //get our next page, we shouldn't get a cursor - final CandidateResults results = entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit); + final CandidateResults results = entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit, false); assertEquals( 0, results.size() ); assertFalse(results.hasOffset()); @@ -733,7 +733,7 @@ public class EntityIndexTest extends BaseIT { final String query = "where searchUUID = " + searchUUID; final CandidateResults r = - entityIndex.search( indexSCope, SearchTypes.fromTypes(entityId.getType()), query, 10, 0); + entityIndex.search( indexSCope, SearchTypes.fromTypes(entityId.getType()), query, 10, 0, false); assertEquals(user.getId(), r.get(0).getId()); } @@ -772,7 +772,7 @@ public class EntityIndexTest extends BaseIT { final String query = "where string = 'I am*'"; final CandidateResults r = - entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), query, 10, 0); + entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), query, 10, 0, false); assertEquals(user.getId(), r.get(0).getId()); @@ -780,7 +780,7 @@ public class EntityIndexTest extends BaseIT { final String queryNoWildCard = "where string = 'I am'"; final CandidateResults noWildCardResults = - entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), queryNoWildCard, 10, 0 ); + entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), queryNoWildCard, 10, 0, false ); assertEquals( 0, noWildCardResults.size() ); } @@ -830,7 +830,7 @@ public class EntityIndexTest extends BaseIT { final String ascQuery = "order by string"; final CandidateResults ascResults = - entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), ascQuery, 10 , 0); + entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), ascQuery, 10 , 0, false); assertEquals( first.getId(), ascResults.get( 0).getId() ); @@ -841,7 +841,7 @@ public class EntityIndexTest extends BaseIT { final String descQuery = "order by string desc"; final CandidateResults descResults = - entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), descQuery, 10 , 0); + entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), descQuery, 10 , 0, false); assertEquals( second.getId(), descResults.get( 0).getId() ); @@ -895,7 +895,7 @@ public class EntityIndexTest extends BaseIT { final String singleMatchQuery = "string contains 'alpha' OR string contains 'foo'"; final CandidateResults singleResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), singleMatchQuery, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), singleMatchQuery, 10, 0, false ); assertEquals(1, singleResults.size()); @@ -906,7 +906,7 @@ public class EntityIndexTest extends BaseIT { final String bothKeywordsMatch = "string contains 'alpha' OR string contains 'bravo'"; final CandidateResults singleKeywordUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), bothKeywordsMatch, 10 , 0); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), bothKeywordsMatch, 10 , 0, false); assertEquals( 2, singleKeywordUnion.size() ); @@ -917,7 +917,7 @@ public class EntityIndexTest extends BaseIT { final String twoKeywordMatches = "string contains 'alpha' OR string contains 'long'"; final CandidateResults towMatchResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), twoKeywordMatches, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), twoKeywordMatches, 10, 0, false ); assertEquals( 2, towMatchResults.size() ); @@ -976,7 +976,7 @@ public class EntityIndexTest extends BaseIT { final String notFirst = "NOT int = 1"; final CandidateResults notFirstResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirst, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirst, 10, 0, false ); assertEquals( 1, notFirstResults.size() ); @@ -987,7 +987,7 @@ public class EntityIndexTest extends BaseIT { final String notSecond = "NOT int = 2"; final CandidateResults notSecondUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecond, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecond, 10, 0, false ); assertEquals( 1, notSecondUnion.size() ); @@ -997,7 +997,7 @@ public class EntityIndexTest extends BaseIT { final String notBothReturn = "NOT int = 3"; final CandidateResults notBothReturnResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notBothReturn, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notBothReturn, 10, 0, false ); assertEquals( 2, notBothReturnResults.size() ); @@ -1008,7 +1008,7 @@ public class EntityIndexTest extends BaseIT { final String notFilterBoth = "(NOT int = 1) AND (NOT int = 2) "; final CandidateResults filterBoth = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFilterBoth, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFilterBoth, 10, 0, false ); assertEquals( 0, filterBoth.size() ); @@ -1016,7 +1016,7 @@ public class EntityIndexTest extends BaseIT { final String noMatchesAnd = "(NOT int = 3) AND (NOT int = 4)"; final CandidateResults noMatchesAndResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesAnd, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesAnd, 10, 0, false ); assertEquals( 2, noMatchesAndResults.size() ); @@ -1027,7 +1027,7 @@ public class EntityIndexTest extends BaseIT { final String noMatchesOr = "(NOT int = 3) AND (NOT int = 4)"; final CandidateResults noMatchesOrResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesOr, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesOr, 10, 0, false ); assertEquals( 2, noMatchesOrResults.size() ); @@ -1086,7 +1086,7 @@ public class EntityIndexTest extends BaseIT { final String notFirst = "NOT string = 'I ate a sammich'"; final CandidateResults notFirstResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirst, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirst, 10, 0, false ); assertEquals( 1, notFirstResults.size() ); @@ -1096,7 +1096,7 @@ public class EntityIndexTest extends BaseIT { final String notFirstWildCard = "NOT string = 'I ate*'"; final CandidateResults notFirstWildCardResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirstWildCard, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirstWildCard, 10, 0, false ); assertEquals( 1, notFirstWildCardResults.size() ); @@ -1106,7 +1106,7 @@ public class EntityIndexTest extends BaseIT { final String notFirstContains = "NOT string contains 'sammich'"; final CandidateResults notFirstContainsResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirstContains, 10 , 0); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirstContains, 10 , 0, false); assertEquals( 1, notFirstContainsResults.size() ); @@ -1117,7 +1117,7 @@ public class EntityIndexTest extends BaseIT { final String notSecond = "NOT string = 'I drank a beer'"; final CandidateResults notSecondUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecond, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecond, 10, 0, false ); assertEquals( 1, notSecondUnion.size() ); @@ -1127,7 +1127,7 @@ public class EntityIndexTest extends BaseIT { final String notSecondWildcard = "NOT string = 'I drank*'"; final CandidateResults notSecondWildcardUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecondWildcard, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecondWildcard, 10, 0, false ); assertEquals( 1, notSecondWildcardUnion.size() ); @@ -1137,7 +1137,7 @@ public class EntityIndexTest extends BaseIT { final String notSecondContains = "NOT string contains 'beer'"; final CandidateResults notSecondContainsUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecondContains, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecondContains, 10, 0, false ); assertEquals( 1, notSecondContainsUnion.size() ); @@ -1147,7 +1147,7 @@ public class EntityIndexTest extends BaseIT { final String notBothReturn = "NOT string = 'I'm a foodie'"; final CandidateResults notBothReturnResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notBothReturn, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notBothReturn, 10, 0, false ); assertEquals( 2, notBothReturnResults.size() ); @@ -1158,7 +1158,7 @@ public class EntityIndexTest extends BaseIT { final String notFilterBoth = "(NOT string = 'I ate a sammich') AND (NOT string = 'I drank a beer') "; final CandidateResults filterBoth = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFilterBoth, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFilterBoth, 10, 0, false ); assertEquals( 0, filterBoth.size() ); @@ -1166,7 +1166,7 @@ public class EntityIndexTest extends BaseIT { final String noMatchesAnd = "(NOT string = 'I ate*') AND (NOT string = 'I drank*')"; final CandidateResults noMatchesAndResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesAnd, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesAnd, 10, 0, false ); assertEquals( 0, noMatchesAndResults.size() ); @@ -1174,7 +1174,7 @@ public class EntityIndexTest extends BaseIT { final String noMatchesContainsAnd = "(NOT string contains 'ate') AND (NOT string contains 'drank')"; final CandidateResults noMatchesContainsAndResults = entityIndex - .search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesContainsAnd, 10, 0 ); + .search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesContainsAnd, 10, 0, false ); assertEquals( 0, noMatchesContainsAndResults.size() ); @@ -1183,7 +1183,7 @@ public class EntityIndexTest extends BaseIT { final String noMatchesOr = "(NOT string = 'I ate*') AND (NOT string = 'I drank*')"; final CandidateResults noMatchesOrResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesOr, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesOr, 10, 0, false ); assertEquals( 0, noMatchesOrResults.size() ); @@ -1191,7 +1191,7 @@ public class EntityIndexTest extends BaseIT { final String noMatchesContainsOr = "(NOT string contains 'ate') AND (NOT string contains 'drank')"; final CandidateResults noMatchesContainsOrResults = entityIndex - .search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesContainsOr, 10, 0 ); + .search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesContainsOr, 10, 0, false ); assertEquals( 0, noMatchesContainsOrResults.size() ); @@ -1236,7 +1236,7 @@ public class EntityIndexTest extends BaseIT { indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); - long size = entityIndex.getEntitySize(new SearchEdgeImpl(ownerId,type, SearchEdge.NodeType.SOURCE)); + long size = entityIndex.getTotalEntitySizeInBytes(new SearchEdgeImpl(ownerId,type, SearchEdge.NodeType.SOURCE)); assertTrue( size == 100 ); } @@ -1289,7 +1289,7 @@ public class EntityIndexTest extends BaseIT { StopWatch timer = new StopWatch(); timer.start(); CandidateResults candidateResults = - entityIndex.search( indexEdge, searchTypes, "select * where uuid = '"+uuid+"'", 100, 0 ); + entityIndex.search( indexEdge, searchTypes, "select * where uuid = '"+uuid+"'", 100, 0, false ); timer.stop(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java index 3d68fe1..ba33030 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java @@ -138,7 +138,7 @@ public class GeoPagingTest extends BaseIT { final String query = "select * where location within 1500000 of 37, -75" ; final CandidateResults - candidates = entityIndex.search( edge, SearchTypes.fromTypes( "cat" ), query, 100, 0 ); + candidates = entityIndex.search( edge, SearchTypes.fromTypes( "cat" ), query, 100, 0, false ); assertNotNull( candidates ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java index afaebb7..a5c3b53 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java @@ -223,7 +223,7 @@ public class IndexLoadTestsIT extends BaseIT { .search( searchEdge, SearchTypes.fromTypes( searchEdge.getEdgeName() ), "select * where " + FIELD_WORKER_INDEX + " = " + workerIndex + " AND " + FIELD_ORDINAL + " = " + ordinal + " AND " + FIELD_UNIQUE_IDENTIFIER + " = '" + uniqueIdentifier - + "'" , 100 , 0); + + "'" , 100 , 0, false); queryTps.mark(); queryTimerContext.stop(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java index 898eb3b..b8c1caa 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java @@ -18,16 +18,10 @@ package org.apache.usergrid.rest.applications; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; -import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -35,7 +29,6 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.PathSegment; -import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import org.springframework.context.annotation.Scope; @@ -44,25 +37,15 @@ import org.springframework.stereotype.Component; import org.apache.commons.lang.NullArgumentException; import org.apache.commons.lang.StringUtils; -import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder; -import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl; -import org.apache.usergrid.corepersistence.index.ReIndexService; import org.apache.usergrid.persistence.Query; -import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException; -import org.apache.usergrid.persistence.index.utils.UUIDUtils; -import org.apache.usergrid.rest.AbstractContextResource; import org.apache.usergrid.rest.ApiResponse; -import org.apache.usergrid.rest.RootResource; -import org.apache.usergrid.rest.exceptions.RequiredPropertyNotFoundExceptionMapper; import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess; import org.apache.usergrid.rest.security.annotations.RequireSystemAccess; import org.apache.usergrid.rest.system.IndexResource; -import org.apache.usergrid.services.AbstractCollectionService; import org.apache.usergrid.services.ServiceAction; import org.apache.usergrid.services.ServiceParameter; import org.apache.usergrid.services.ServicePayload; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java index 91c7db9..3835b75 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java @@ -32,7 +32,6 @@ import org.apache.usergrid.rest.ApiResponse; import org.apache.usergrid.rest.RootResource; import org.apache.usergrid.rest.applications.assets.AssetsResource; import org.apache.usergrid.rest.security.annotations.CheckPermissionsForPath; -import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess; import org.apache.usergrid.security.oauth.AccessInfo; import org.apache.usergrid.services.*; import org.apache.usergrid.services.assets.data.AssetUtils; @@ -262,7 +261,7 @@ public class ServiceResource extends AbstractContextResource { addQueryParams( getServiceParameters(), ui ); ServiceRequest r = services.newRequest( action, tree, getServiceParameters(), payload, - returnInboundConnections, returnOutboundConnections ); + returnInboundConnections, returnOutboundConnections, false); response.setServiceRequest( r ); @@ -316,7 +315,7 @@ public class ServiceResource extends AbstractContextResource { addQueryParams( getServiceParameters(), ui ); ServiceRequest r = services.newRequest( action, tree, getServiceParameters(), payload, - returnInboundConnections, returnOutboundConnections ); + returnInboundConnections, returnOutboundConnections, false); response.setServiceRequest( r ); @@ -400,13 +399,15 @@ public class ServiceResource extends AbstractContextResource { } } + boolean analyzeQueryOnly = Boolean.valueOf(ui.getQueryParameters().getFirst("analyzeOnly")); + boolean collectionGet = false; if ( action == ServiceAction.GET ) { collectionGet = getServiceParameters().size() == 1; } addQueryParams( getServiceParameters(), ui ); ServiceRequest r = services.newRequest( action, tree, getServiceParameters(), payload, - returnInboundConnections, returnOutboundConnections ); + returnInboundConnections, returnOutboundConnections, analyzeQueryOnly); response.setServiceRequest( r ); ServiceResults results = r.execute(); if ( results != null ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java index 4a4b8b0..807e0c9 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java @@ -71,11 +71,11 @@ public abstract class AbstractExceptionMapper<E extends java.lang.Throwable> imp if ( status >= 500 ) { // only log real errors as errors - logger.error( "{} 5XX Uncaught Exception ({})", e.getClass().getCanonicalName(), status, e ); + logger.error( "{} 5XX Uncaught Exception ({}), {}", e.getClass().getCanonicalName(), status, e ); } else { if (logger.isDebugEnabled()) { - logger.debug( "{} Following Exception Thrown ({})", e.getClass().getCanonicalName(), status, e ); + logger.debug( "{} Following Exception Thrown ({}), {}", e.getClass().getCanonicalName(), status, e ); } } @@ -98,12 +98,19 @@ public abstract class AbstractExceptionMapper<E extends java.lang.Throwable> imp public Response toResponse( Status status, String jsonResponse ) { - return toResponse( status.getStatusCode(), jsonResponse ); + return toResponse( status.getStatusCode(), jsonResponse, false ); } + public Response toResponse( Status status, String jsonResponse, boolean skipLogging ) { + return toResponse( status.getStatusCode(), jsonResponse, skipLogging ); + } - protected Response toResponse( int status, String jsonResponse ) { - if ( status >= 500 ) { + public Response toResponse( int statusCode, String jsonResponse ) { + return toResponse( statusCode, jsonResponse, false ); + } + + protected Response toResponse( int status, String jsonResponse, boolean skipLogging ) { + if ( status >= 500 && !skipLogging) { // only log real errors as errors logger.error( "Server Error ({}):\n{}", status, jsonResponse ); } else if ( logger.isDebugEnabled() ) {