Working lower level selective indexing. Working field updates when we do reindexing and stuff. Added test proving that it works.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b5d81ac0 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b5d81ac0 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b5d81ac0 Branch: refs/heads/release-2.1.1 Commit: b5d81ac0334fc5aeb204f137bfcbd500c7c1086c Parents: 82ff53b Author: George Reyes <g...@apache.org> Authored: Tue Mar 29 14:11:56 2016 -0700 Committer: George Reyes <g...@apache.org> Committed: Tue Mar 29 14:11:56 2016 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 18 ++- .../corepersistence/index/IndexServiceImpl.java | 52 ++------ .../index/ReIndexServiceImpl.java | 26 +++- .../persistence/index/EntityIndexBatch.java | 13 +- .../index/impl/EntityToMapConverter.java | 133 ++++++++++++++++++- .../index/impl/EsEntityIndexBatchImpl.java | 27 ++-- .../persistence/index/impl/IndexOperation.java | 20 +-- .../persistence/index/impl/EntityIndexTest.java | 4 +- .../collection/CollectionsResourceIT.java | 25 +++- .../test/resource/endpoints/NamedResource.java | 22 +++ 10 files changed, 242 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index bb3abcb..2c97bc6 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -1737,7 +1737,6 @@ public class CpEntityManager implements EntityManager { @Override public Map createCollectionSchema( String collectionName, String owner ,Map<String, Object> properties ){ - //haven't decided which one I should base off of which, maybe from epoch to utc //TODO: change timeservice as below then use timeservice. @@ -1753,10 +1752,23 @@ public class CpEntityManager implements EntityManager { //this needs the method that can extract the user from the token no matter the token. //Possible values are app credentials, org credentials, or the user email(Admin tokens). schemaMap.put("lastUpdateBy",owner); - schemaMap.put("lastReindexed",0); - schemaMap.putAll( properties ); + + //TODO: please add a check to get the previous reindex time. MapManager mm = getMapManagerForTypes(); + + String jsonSchemaMap = mm.getString( collectionName ); + + //If we do have a schema then parse it and add it to a list of properties we want to keep.Otherwise return. + if ( jsonSchemaMap != null ) { + Map jsonMapData = ( Map ) JsonUtils.parse( jsonSchemaMap ); + schemaMap.put( "lastReindexed", jsonMapData.get( "lastReindexed" ) ); + } + else { + schemaMap.put( "lastReindexed", 0 ); + } + schemaMap.putAll( properties ); + mm.putString( collectionName,JsonUtils.mapToJsonString( schemaMap ) ); return schemaMap; http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java index 5f50a32..c3d5361 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -134,14 +135,9 @@ public class IndexServiceImpl implements IndexService { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); } - final Map map = getFilteredStringObjectMap( applicationScope, entity, indexEdge ); + final Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( applicationScope, entity, indexEdge ); - if(map!=null){ - batch.index( indexEdge, entity ,map); - } - else{ - batch.index( indexEdge,entity ); - } + batch.index( indexEdge, entity ,fieldsToIndex); } ) //return the future from the batch execution .map( batch -> batch.build() ) ); @@ -172,14 +168,9 @@ public class IndexServiceImpl implements IndexService { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); } - Map map = getFilteredStringObjectMap( applicationScope, entity, indexEdge ); + Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( applicationScope, entity, indexEdge ); - if(map!=null){ - batch.index( indexEdge, entity ,map); - } - else{ - batch.index( indexEdge,entity ); - } + batch.index( indexEdge, entity ,fieldsToIndex); return batch.build(); @@ -202,9 +193,11 @@ public class IndexServiceImpl implements IndexService { * @return This returns a filtered map that contains the flatted properties of the entity. If there isn't a schema * associated with the collection then return null ( and index the entity in its entirety ) */ - private Map getFilteredStringObjectMap( final ApplicationScope applicationScope, + private Optional<Set<String>> getFilteredStringObjectMap( final ApplicationScope applicationScope, final Entity entity, final IndexEdge indexEdge ) { + + //TODO: THIS IS THE FIRST THING TO BE LOOKED AT TOMORROW. //look into this. IndexOperation indexOperation = new IndexOperation(); @@ -233,35 +226,10 @@ public class IndexServiceImpl implements IndexService { defaultProperties.addAll( fieldsToKeep ); } else { - return null; - } - - //Returns the flattened map of the entity. - //TODO: maybe instead pass the fields to keep to the flattening. - Map map = indexOperation.convertedEntityToBeIndexed( applicationScope, indexEdge, entity ); - - HashSet mapFields = ( HashSet ) map.get( "fields" ); - Iterator collectionIterator = mapFields.iterator(); - - //Loop through all of the fields of the flatted entity and check to see if they should be filtered out. - while ( collectionIterator.hasNext() ) { - EntityField testedField = ( EntityField ) collectionIterator.next(); - String fieldName = ( String ) ( testedField ).get( "name" ); - - //Checks to see if the fieldname is a default property. If it is then keep it, otherwise send it to - //be verified the aptly named method - - //one.two.three - //one.two.four - //one.two3.five - //one.two - //fields { one.two } - if ( !defaultProperties.contains( fieldName ) ) { - iterateThroughMapForFieldsToBeIndexed( fieldsToKeep, collectionIterator, fieldName ); - } + return Optional.empty(); } - return map; + return Optional.of(defaultProperties); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java index 558a18c..42067f5 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java @@ -20,7 +20,9 @@ package org.apache.usergrid.corepersistence.index; +import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; @@ -42,7 +44,10 @@ import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.map.MapManagerFactory; import org.apache.usergrid.persistence.map.MapScope; import org.apache.usergrid.persistence.map.impl.MapScopeImpl; +import org.apache.usergrid.persistence.model.entity.SimpleId; import org.apache.usergrid.persistence.model.util.UUIDGenerator; +import org.apache.usergrid.utils.InflectionUtils; +import org.apache.usergrid.utils.JsonUtils; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Optional; @@ -76,6 +81,7 @@ public class ReIndexServiceImpl implements ReIndexService { private final AllEntityIdsObservable allEntityIdsObservable; private final IndexProcessorFig indexProcessorFig; private final MapManager mapManager; + private final MapManagerFactory mapManagerFactory; private final AsyncEventService indexService; private final EntityIndexFactory entityIndexFactory; @@ -94,7 +100,7 @@ public class ReIndexServiceImpl implements ReIndexService { this.allApplicationsObservable = allApplicationsObservable; this.indexProcessorFig = indexProcessorFig; this.indexService = indexService; - + this.mapManagerFactory = mapManagerFactory; this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPE ); } @@ -133,12 +139,30 @@ public class ReIndexServiceImpl implements ReIndexService { .buffer( indexProcessorFig.getReindexBufferSize()); if(delayTimer.isPresent()){ + + if(reIndexRequestBuilder.getCollectionName().isPresent()) { + String collectionName = InflectionUtils.pluralize( CpNamingUtils.getNameFromEdgeType(reIndexRequestBuilder.getCollectionName().get() )); + MapManager collectionMapStorage = mapManagerFactory.createMapManager( CpNamingUtils.getEntityTypeMapScope( appId.get().getApplication() ) ); + String jsonSchemaMap = collectionMapStorage.getString( collectionName ); + + + //If we do have a schema then parse it and add it to a list of properties we want to keep.Otherwise return. + if ( jsonSchemaMap != null ) { + + Map jsonMapData = ( Map ) JsonUtils.parse( jsonSchemaMap ); + + jsonMapData.put( "lastReindexed", Instant.now().toEpochMilli() ); + collectionMapStorage.putString( collectionName,JsonUtils.mapToJsonString(jsonMapData ) ); + } + + } if(timeUnitOptional.isPresent()){ runningReIndex = runningReIndex.delay( delayTimer.get(),timeUnitOptional.get() ); } else{ runningReIndex = runningReIndex.delay( delayTimer.get(), TimeUnit.MILLISECONDS ); } + } runningReIndex = runningReIndex.doOnNext(edges -> { http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java index e3a40f6..2e5ece6 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java @@ -20,6 +20,8 @@ package org.apache.usergrid.persistence.index;/* import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.UUID; import org.apache.usergrid.persistence.index.impl.IndexOperationMessage; @@ -30,13 +32,7 @@ import rx.Observable; public interface EntityIndexBatch { - /** - * Create index for Entity - * - * @param indexEdge The edge to index the document into - * @param entity Entity to be indexed. - */ - EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity ); + EntityIndexBatch index( IndexEdge indexEdge, Entity entity ); /** * Remove index of entity @@ -54,7 +50,8 @@ public interface EntityIndexBatch { */ EntityIndexBatch deindex( final SearchEdge searchEdge, final CandidateResult result ); - EntityIndexBatch index( IndexEdge indexEdge, Entity entity, Map flattenedEntityMap ); + + EntityIndexBatch index( IndexEdge indexEdge, Entity entity, Optional<Set<String>> fieldsToIndex ); /** * Remove index of entity. http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java index b43f4bb..5a728a3 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java @@ -17,8 +17,12 @@ package org.apache.usergrid.persistence.index.impl; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.apache.usergrid.persistence.core.scope.ApplicationScope; @@ -48,6 +52,14 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.nodeId; * Convert a CP entity to an elasticsearch document */ public class EntityToMapConverter { + + + public static Map<String, Object> convert(ApplicationScope applicationScope, final IndexEdge indexEdge, + final Entity entity) { + + return convert( applicationScope, indexEdge, entity, Optional.empty() ); + } + /** * Set the entity as a map with the context * @@ -55,7 +67,8 @@ public class EntityToMapConverter { * @param entity The entity * @param indexEdge The edge this entity is indexed on */ - public static Map<String, Object> convert(ApplicationScope applicationScope, final IndexEdge indexEdge, final Entity entity) { + public static Map<String, Object> convert(ApplicationScope applicationScope, final IndexEdge indexEdge, + final Entity entity, Optional<Set<String>> fieldsToIndex) { @@ -95,14 +108,128 @@ public class EntityToMapConverter { //now visit our entity final FieldParser parser = new EntityMappingParser(); - final Set<EntityField> fieldsToIndex = parser.parse( entityMap ); + //TODO: to the parsing in the parse method below, that way we don't iterate twice and we filter things out as we see them. + final Set<EntityField> fieldsToBeFiltered = parser.parse( entityMap ); //add our fields - outputEntity.put( ENTITY_FIELDS, fieldsToIndex ); + outputEntity.put( ENTITY_FIELDS, fieldsToBeFiltered ); + + if(fieldsToIndex.isPresent()){ + Set<String> defaultProperties = fieldsToIndex.get(); + //copy paste logic here. + HashSet mapFields = ( HashSet ) outputEntity.get( "fields" ); + Iterator collectionIterator = mapFields.iterator(); + + //Loop through all of the fields of the flatted entity and check to see if they should be filtered out. + while ( collectionIterator.hasNext() ) { + EntityField testedField = ( EntityField ) collectionIterator.next(); + String fieldName = ( String ) ( testedField ).get( "name" ); + + //Checks to see if the fieldname is a default property. If it is then keep it, otherwise send it to + //be verified the aptly named method + + //one.two.three + //one.two.four + //one.two3.five + //one.two + //fields { one.two } + if ( !defaultProperties.contains( fieldName ) ) { + iterateThroughMapForFieldsToBeIndexed( defaultProperties, collectionIterator, fieldName ); + } + } + } + return outputEntity; } + /** + * This method is crucial for selective top level indexing. Here we check to see if the flatted properties + * are in fact a top level exclusion e.g one.two.three and one.three.two can be allowed for querying by + * specifying one in the schema. If they are not a top level exclusion then they are removed from the iterator and + * the map. + * + * @param fieldsToKeep - contains a list of fields that the user defined in their schema. + * @param collectionIterator - contains the iterator with the reference to the map where we want to remove the field. + * @param fieldName - contains the name of the field that we want to keep. + */ + private static void iterateThroughMapForFieldsToBeIndexed( final Set<String> fieldsToKeep, + final Iterator collectionIterator, + final String fieldName ) { + boolean toRemoveFlag = true; + String[] flattedStringArray = getStrings( fieldName ); + + Iterator fieldIterator = fieldsToKeep.iterator(); //fieldsToKeep.iterator(); + + //goes through a loop of all the fields ( excluding default ) that we want to keep. + //if the toRemoveFlag is set to false then we want to keep the property, otherwise we set it to true and remove + //the property. + while ( fieldIterator.hasNext() ) { + String requiredInclusionString = ( String ) fieldIterator.next(); + String[] flattedRequirementString = getStrings( requiredInclusionString ); + + + //loop each split array value to see if it matches the equivalent value + //in the field. e.g in the example one.two.three and one.two.four we need to check that the schema + //matches in both one and two above. If instead it says to exclude one.twor then we would still exclude the above + //since it is required to be a hard match. + + //The way the below works if we see that the current field isn't as fine grained as the schema rule + //( aka the array is shorter than the current index of the schema rule then there is no way the rule could apply + // to the index. + + //Then if that check passes we go to check that both parts are equal. If they are ever not equal + // e.g one.two.three and one.three.two then it shouldn't be included + //TODO: regex. + for ( int index = 0; index < flattedRequirementString.length; index++ ) { + //if the array contains a string that it is equals to then set the remove flag to true + //otherwise remain false. + + //one.three + //one.two + //one + if ( flattedStringArray.length <= index ) { + toRemoveFlag = true; + break; + } + + if ( flattedRequirementString[index].equals( flattedStringArray[index] ) ) { + toRemoveFlag = false; + } + else { + toRemoveFlag = true; + break; + } + } + if ( toRemoveFlag == false ) { + break; + } + } + + if ( toRemoveFlag ) { + collectionIterator.remove(); + } + } + + + /** + * Splits the string on the flattened period "." seperated values. + * @param fieldName + * @return + */ + private static String[] getStrings( final String fieldName ) { + final String[] flattedStringArray; + if ( !fieldName.contains( "." ) ) { + //create a single array that is made of a the single value. + flattedStringArray = new String[] { fieldName }; + } + else { + flattedStringArray = fieldName.split( "\\." ); + } + return flattedStringArray; + } + } + http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index 5296978..ed00f48 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -20,12 +20,16 @@ package org.apache.usergrid.persistence.index.impl; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.UUID; import org.apache.usergrid.persistence.index.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.auth.IAuthenticator; + import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; import org.apache.usergrid.persistence.index.utils.IndexValidationUtils; @@ -65,24 +69,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { @Override public EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity ) { - IndexValidationUtils.validateIndexEdge(indexEdge); - ValidationUtils.verifyEntityWrite(entity); - ValidationUtils.verifyVersion( entity.getVersion() ); - - final String writeAlias = alias.getWriteAlias(); - - if ( logger.isDebugEnabled() ) { - logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data {}", - writeAlias, applicationScope, indexEdge, entity.getFieldMap().keySet() ); - } - - //add app id for indexing - container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity)); - return this; + return index( indexEdge,entity, Optional.empty() ); } @Override - public EntityIndexBatch index ( final IndexEdge indexEdge, final Entity entity ,final Map flattenedEntityMap){ + public EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity, final Optional<Set<String>> fieldsToIndex ) { IndexValidationUtils.validateIndexEdge(indexEdge); ValidationUtils.verifyEntityWrite(entity); ValidationUtils.verifyVersion( entity.getVersion() ); @@ -91,16 +82,14 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { if ( logger.isDebugEnabled() ) { logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data {}", - writeAlias, applicationScope, indexEdge, flattenedEntityMap ); + writeAlias, applicationScope, indexEdge, entity.getFieldMap().keySet() ); } //add app id for indexing - container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, - indexEdge,entity.getId(),entity.getVersion(),flattenedEntityMap) ); + container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity,fieldsToIndex)); return this; } - @Override public EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID version ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java index b51d0c7..8f925ed 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java @@ -21,6 +21,8 @@ package org.apache.usergrid.persistence.index.impl; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.UUID; import org.apache.usergrid.persistence.core.scope.ApplicationScope; @@ -49,21 +51,13 @@ public class IndexOperation implements BatchOperation { public Map<String, Object> data; public IndexOperation( final String writeAlias, final ApplicationScope applicationScope, IndexEdge indexEdge, - Entity entity ) { - - - this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge) - ,EntityToMapConverter.convert(applicationScope,indexEdge, entity)); + Entity entity, Optional<Set<String>> fieldsToIndex ) { + this( writeAlias, IndexingUtils.createIndexDocId( applicationScope, entity, indexEdge ), + EntityToMapConverter.convert( applicationScope, indexEdge, entity, fieldsToIndex ) ); } - public IndexOperation( final String writeAlias, final ApplicationScope applicationScope, IndexEdge indexEdge, - Id entityId, UUID version, Map<String,Object> data ) { - - this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entityId,version,indexEdge) - ,data); - } public IndexOperation( final String writeAlias, String documentId, Map<String, Object> data ) { this.writeAlias = writeAlias; @@ -120,8 +114,4 @@ public class IndexOperation implements BatchOperation { result = 31 * result + data.hashCode(); return result; } - - public Map convertedEntityToBeIndexed(ApplicationScope applicationScope,IndexEdge indexEdge,Entity entity){ - return EntityToMapConverter.convert(applicationScope,indexEdge, entity); - } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index 3978956..27f3afd 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -365,7 +365,7 @@ public class EntityIndexTest extends BaseIT { EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID() ); entity.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) ); - indexProducer.put(entityIndex.createBatch().index( searchEdge, entity ).build()).subscribe(); + indexProducer.put(entityIndex.createBatch().index( searchEdge, entity, null ).build()).subscribe(); entityIndex.refreshAsync().toBlocking().first(); CandidateResults candidateResults = entityIndex @@ -412,7 +412,7 @@ public class EntityIndexTest extends BaseIT { IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, System.currentTimeMillis()*1000 ); //index the new entity. This is where the loop will be set to create like 100 entities. - indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i] ).build()).subscribe(); + indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i], null ).build()).subscribe(); } entityIndex.refreshAsync().toBlocking().first(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java index 109bb6f..7926e49 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java @@ -28,6 +28,7 @@ import org.apache.usergrid.rest.test.resource.model.Collection; import org.apache.usergrid.rest.test.resource.model.Entity; import org.apache.usergrid.rest.test.resource.model.QueryParameters; import org.apache.usergrid.rest.test.resource.model.Token; +import org.apache.usergrid.services.ServiceParameter; import org.junit.Ignore; import org.junit.Test; @@ -167,7 +168,7 @@ public class CollectionsResourceIT extends AbstractRestIT { //Reindex and verify that the entity only has field one index. - this.app().collection( "testCollection" ).collection( "_reindex" ).post(); + this.app().collection( "testCollection" ).collection( "_reindex" ).post(true,clientSetup.getSuperuserToken(),ApiResponse.class,null,null,false); // Thread.sleep( 10000 ); //refreshIndex(); @@ -225,7 +226,7 @@ public class CollectionsResourceIT extends AbstractRestIT { //TODO: the below will have to be replaced by the values that I deem correct. assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated" )); assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy" ) ); - assertEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed" ) ); + assertEquals( 0,testCollectionSchema.get( "lastReindexed" ) ); //TODO: this test doesn't check to see if create checks the schema. Only that the reindex removes whats already there. ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" ); @@ -233,14 +234,28 @@ public class CollectionsResourceIT extends AbstractRestIT { //Reindex and verify that the entity only has field one index. - this.app().collection( "testCollection" ).collection( "_reindex" ).post(); + this.app().collection( "testCollection" ).collection( "_reindex" ).post(true,clientSetup.getSuperuserToken(),ApiResponse.class,null,null,false); + + + indexingArray.add( "one" ); + indexingArray.add( "two" ); + + + //field "fields" is required. + payload = new Entity(); + payload.put( "fields", indexingArray); + + //Post index to the collection metadata + this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload ); collection = this.app().collection( "testCollection" ).collection( "_index" ).get(); + + testCollectionSchema = (LinkedHashMap)collection.getResponse().getData(); - assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated" )); + assertNotEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated" )); assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy" ) ); - assertNotEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed" ) ); + assertNotEquals( 0,testCollectionSchema.get( "lastReindexed" ) ); schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" ); assertEquals( "one",schema.get( 0 ) ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java index 0254d18..a2377d4 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java @@ -272,6 +272,28 @@ public class NamedResource implements UrlResource { .post(javax.ws.rs.client.Entity.json(entity), gt); } + // Used for empty posts + public <T> T post(boolean useToken, Token tokenToUse , Class<T> type, Map entity, + final QueryParameters queryParameters, boolean useBasicAuthentication) { + + WebTarget resource = getTarget( useToken,tokenToUse ); + resource = addParametersToResource(resource, queryParameters); + + GenericType<T> gt = new GenericType<>((Class) type); + + if (useBasicAuthentication) { + HttpAuthenticationFeature feature = HttpAuthenticationFeature.basicBuilder() + .credentials("superuser", "superpassword").build(); + return resource.register(feature).request() + .accept(MediaType.APPLICATION_JSON) + .post(javax.ws.rs.client.Entity.json(entity), gt); + } + + return resource.request() + .accept(MediaType.APPLICATION_JSON) + .post(javax.ws.rs.client.Entity.json(entity), gt); + } + //For edge cases like Organizations and Tokens without any payload public <T> T get(Class<T> type) { return get(type, null, true);
