Disable indexing per Entity type for USERGRID-1278. https://issues.apache.org/jira/browse/USERGRID-1278
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fbced588 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fbced588 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fbced588 Branch: refs/heads/master Commit: fbced588bdf7de5f7192d20e2da6a8740df0218a Parents: 2e576f3 Author: Dave Johnson <snoopd...@apache.org> Authored: Thu Apr 14 12:26:06 2016 -0400 Committer: Dave Johnson <snoopd...@apache.org> Committed: Thu Apr 14 12:26:06 2016 -0400 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 86 ++++++++++++++------ .../collection/CollectionsResourceIT.java | 56 +++++++++++-- 2 files changed, 112 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbced588/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 891c1fe..b2330f3 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -601,14 +601,36 @@ public class CpEntityManager implements EntityManager { handleWriteUniqueVerifyException( entity, wuve ); } - // queue an event to update the new entity - indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 ); + if ( !skipIndexingForType( cpEntity.getId().getType() ) ) { + // queue an event to update the new entity + indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 ); - // queue up an event to clean-up older versions than this one from the index - if(entityManagerFig.getDeindexOnUpdate()) { - indexService.queueDeIndexOldVersion(applicationScope, entityId); + // queue up an event to clean-up older versions than this one from the index + if (entityManagerFig.getDeindexOnUpdate()) { + indexService.queueDeIndexOldVersion( applicationScope, entityId ); + } + } + } + + private boolean skipIndexingForType( String type ) { + + boolean skipIndexing = false; + + MapManager mm = getMapManagerForTypes(); + IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm ); + String collectionName = Schema.defaultCollectionName( type ); + Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName ); + + if ( collectionIndexingSchema.isPresent()) { + Map jsonMapData = collectionIndexingSchema.get(); + final ArrayList fields = (ArrayList) jsonMapData.get( "fields" ); + if ( fields.size() == 1 && fields.get(0).equals("!")) { + skipIndexing = true; + } } + + return skipIndexing; } @@ -670,8 +692,9 @@ public class CpEntityManager implements EntityManager { Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() ); - //Step 4 && 5 - indexService.queueEntityDelete( applicationScope, entityId ); + if ( !skipIndexingForType( entityId.getType() ) ) { + indexService.queueEntityDelete( applicationScope, entityId ); + } //Step 6 //delete from our UUID index @@ -714,7 +737,8 @@ public class CpEntityManager implements EntityManager { } @Override - public Results searchCollectionConsistent( EntityRef entityRef, String collectionName, Query query, int expectedResults) throws Exception { + public Results searchCollectionConsistent( + EntityRef entityRef, String collectionName, Query query, int expectedResults) throws Exception { return getRelationManager( entityRef ).searchCollectionConsistent(collectionName, query, expectedResults); } @@ -754,8 +778,8 @@ public class CpEntityManager implements EntityManager { public RelationManager getRelationManager( EntityRef entityRef ) { Preconditions.checkNotNull(entityRef, "entityRef cannot be null"); - CpRelationManager relationManager = - new CpRelationManager(managerCache, indexService, collectionService, connectionService, this, entityManagerFig, applicationId, entityRef ); + CpRelationManager relationManager = new CpRelationManager( managerCache, indexService, collectionService, + connectionService, this, entityManagerFig, applicationId, entityRef ); return relationManager; } @@ -830,7 +854,8 @@ public class CpEntityManager implements EntityManager { Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time(); - //TODO: can't we just sub in the getEntityRepair method here so for every read of a uniqueEntityField we can verify it is correct? + // TODO: can't we just sub in the getEntityRepair method here + // so for every read of a uniqueEntityField we can verify it is correct? StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString()); @@ -916,8 +941,8 @@ public class CpEntityManager implements EntityManager { // add a warn statement so we can see if we have data migration issues. // TODO When we get an event system, trigger a repair if this is detected if ( results.size() > 1 ) { - logger.warn( "More than 1 entity with Owner id '{}' of type '{}' and alias '{}' exists. This is a duplicate alias, and needs audited", - ownerRef, collectionType, aliasValue ); + logger.warn( "More than 1 entity with Owner id '{}' of type '{}' and alias '{}' exists. " + + "This is a duplicate alias, and needs audited", ownerRef, collectionType, aliasValue ); } return results.get(aliasValue); @@ -1139,7 +1164,9 @@ public class CpEntityManager implements EntityManager { //Adding graphite metrics - indexService.queueEntityIndexUpdate(applicationScope, cpEntity, 0); + if ( !skipIndexingForType( cpEntity.getId().getType() ) ) { + indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 ); + } } @@ -1785,12 +1812,17 @@ public class CpEntityManager implements EntityManager { //do a check to see if you have a * field. If you do have a * field then ignore all other fields //and only accept the * field. - if(fieldProperties.contains( "*" )){ - ArrayList<String> wildCardArrayList = new ArrayList<>( ); + if ( fieldProperties.contains( "*" )) { + ArrayList<String> wildCardArrayList = new ArrayList<>(); wildCardArrayList.add( "*" ); - schemaMap.put( "fields",wildCardArrayList ); - } - else { + schemaMap.put( "fields", wildCardArrayList ); + + } else if ( fieldProperties.contains( "!" )) { + ArrayList<String> wildCardArrayList = new ArrayList<>(); + wildCardArrayList.add( "!" ); + schemaMap.put( "fields", wildCardArrayList ); + + } else { schemaMap.putAll( properties ); } @@ -1815,7 +1847,7 @@ public class CpEntityManager implements EntityManager { public Object getCollectionSchema( String collectionName ){ MapManager mm = getMapManagerForTypes(); - IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm ); //managerCache.getIndexSchema( mm ); + IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm ); Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName ); @@ -2388,7 +2420,8 @@ public class CpEntityManager implements EntityManager { if ( !skipAggregateCounters ) { long timestamp = cass.createTimestamp(); Mutator<ByteBuffer> m = createMutator( cass.getApplicationKeyspace( applicationId ), be ); - counterUtils.batchIncrementAggregateCounters( m, applicationId, userId, groupId, null, category, counters, timestamp ); + counterUtils.batchIncrementAggregateCounters( + m, applicationId, userId, groupId, null, category, counters, timestamp ); //Adding graphite metrics Timer.Context timeIncrementCounters =entIncrementAggregateCountersTimer.time(); @@ -2987,8 +3020,9 @@ public class CpEntityManager implements EntityManager { final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId() ); - return graphManager.getEdgeTypesFromSource( searchByEdgeType ).map( edgeName -> getConnectionNameFromEdgeName( edgeName ) ).collect( - () -> new HashSet<String>(), ( r, s ) -> r.add( s ) ).toBlocking().last(); + return graphManager.getEdgeTypesFromSource( + searchByEdgeType ).map( edgeName -> getConnectionNameFromEdgeName( edgeName ) ) + .collect( () -> new HashSet<String>(), ( r, s ) -> r.add( s ) ).toBlocking().last(); } @@ -3001,7 +3035,8 @@ public class CpEntityManager implements EntityManager { final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId() ); return graphManager.getEdgeTypesToTarget(searchByEdgeType).map( - edgeName -> getConnectionNameFromEdgeName( edgeName ) ).collect( () -> new HashSet<String>( ), ( r, s ) -> r.add(s) ).toBlocking().last(); + edgeName -> getConnectionNameFromEdgeName( edgeName ) ) + .collect( () -> new HashSet<String>( ), ( r, s ) -> r.add(s) ).toBlocking().last(); } @@ -3030,7 +3065,8 @@ public class CpEntityManager implements EntityManager { try { for (int i = 0; i < 20; i++) { if (searchCollection( - new SimpleEntityRef(org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE, getApplicationId()), + new SimpleEntityRef( + org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE, getApplicationId()), InflectionUtils.pluralize("refresh"), Query.fromQL("select * where uuid='" + refreshEntity.getUuid() + "'") ).size() > 0 http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbced588/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java index d052564..b240629 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java @@ -19,6 +19,7 @@ package org.apache.usergrid.rest.applications.collection; import com.fasterxml.jackson.databind.JsonNode; +import org.apache.commons.lang.RandomStringUtils; import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.entities.Application; @@ -42,11 +43,7 @@ import javax.ws.rs.client.Invocation; import javax.ws.rs.core.MediaType; import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; -import java.util.ArrayList; +import java.util.*; import org.apache.commons.lang.NullArgumentException; @@ -943,4 +940,53 @@ public class CollectionsResourceIT extends AbstractRestIT { Assert.assertEquals( "Should not be more than one name property", -1, secondFred ); */ } + + + /** + * Test that when schema is "!" entity gets saved but does not get indexed + */ + @Test + public void postCollectionSchemaWithWildcardIndexNone() throws Exception { + + // creating schema with no index wildcard and other fields that should be ignored + ArrayList<String> indexingArray = new ArrayList<>( ); + indexingArray.add( "!" ); + indexingArray.add( "one" ); + indexingArray.add( "two" ); + Entity payload = new Entity(); + payload.put( "fields", indexingArray); + + String randomizer = RandomStringUtils.randomAlphanumeric(10); + String collectionName = "col_" + randomizer; + app().collection( collectionName ).collection( "_indexes" ).post( payload ); + refreshIndex(); + + // was the no-index wildcard saved and others ignored? + Collection collection = app().collection( collectionName ).collection( "_index" ).get(); + LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData(); + ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" ); + assertTrue( schema.contains( "!" ) ); + assertFalse( schema.contains( "one" ) ); + assertFalse( schema.contains( "two" ) ); + + // post an entity with a name and a color + String entityName = "name_" + randomizer; + Entity postedEntity = this.app().collection( collectionName ) + .post( new Entity().withProp("name", entityName).withProp( "color", "magenta" ) ); + + // should be able to get entity by ID + Entity getByIdEntity = app().collection( collectionName ).entity( postedEntity.getUuid() ).get(); + assertNotNull( getByIdEntity ); + assertEquals( postedEntity.getUuid(), getByIdEntity.getUuid() ); + + // should be able to get entity by name + Entity getByNameEntity = app().collection( collectionName ).entity( entityName ).get(); + assertNotNull( getByNameEntity ); + assertEquals( postedEntity.getUuid(), getByNameEntity.getUuid() ); + + // should NOT be able to get entity by query + Iterator<Entity> getByQuery = app().collection( collectionName ) + .get( new QueryParameters().setQuery( "select * where color='magenta'" ) ).iterator(); + assertFalse( getByQuery.hasNext() ); + } }