Repository: usergrid Updated Branches: refs/heads/master 8b945c638 -> 2110bfbd4
add "replace_all" to POST settings to allow merging of setting properties Modified unit tests to allow for a more flexible response times Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2110bfbd Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2110bfbd Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2110bfbd Branch: refs/heads/master Commit: 2110bfbd44f0b3dd8fcc9ec7cdf9c8c287428ff6 Parents: 8b945c6 Author: Peter Johnson <pjohn...@apigee.com> Authored: Thu Nov 16 13:17:33 2017 -0800 Committer: Peter Johnson <pjohn...@apigee.com> Committed: Thu Nov 16 16:37:13 2017 -0800 ---------------------------------------------------------------------- .../persistence/CollectionDeleteTest.java | 33 +- .../usergrid/persistence/RebuildIndexTest.java | 349 ++++++++++--------- .../rest/applications/CollectionResource.java | 38 +- .../collection/CollectionsResourceIT.java | 83 ++++- .../collection/users/UserResourceIT.java | 33 +- .../resource/endpoints/CollectionEndpoint.java | 30 +- .../AbstractServiceNotificationIT.java | 10 +- .../apns/NotificationsServiceIT.java | 16 +- .../gcm/NotificationsServiceIT.java | 9 +- 9 files changed, 396 insertions(+), 205 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java index 42afa67..9e5598c 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java @@ -130,9 +130,7 @@ public class CollectionDeleteTest extends AbstractCoreIT { } entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) ); - if ( i % 10 == 0 ) { - logger.info( "Created {} entities after delete time", i ); - } + logger.info( "Created {} entities after delete time created time {} ", i , entity.getCreated()); } logger.info("Created {} entities after delete time", ENTITIES_TO_ADD_AFTER_TIME); @@ -155,11 +153,11 @@ public class CollectionDeleteTest extends AbstractCoreIT { waitForDelete( status, collectionDeleteService ); - app.waitForQueueDrainAndRefreshIndex(15000); + //app.waitForQueueDrainAndRefreshIndex(15000); // ----------------- test that we can read the entries after the timestamp - readData( em, collectionName,ENTITIES_TO_ADD_AFTER_TIME); + retryReadData( em, collectionName, ENTITIES_TO_ADD_AFTER_TIME, 60); } /** @@ -199,6 +197,19 @@ public class CollectionDeleteTest extends AbstractCoreIT { } } + private int retryReadData(EntityManager em, String collectionName, int expectedEntities, int retry) throws Exception { + int count = -1; + do { + try { + count = readData(em, collectionName, expectedEntities); + } catch (Exception ignore) { + logger.info( "caught exception ", ignore); + } + logger.info( "read {} expected {}" , count, expectedEntities); + } while (count != expectedEntities && --retry >=0); + assertEquals( "Did not get expected entities", expectedEntities, count ); + return count; + } private int readData(EntityManager em, String collectionName, int expectedEntities) throws Exception { @@ -209,6 +220,7 @@ public class CollectionDeleteTest extends AbstractCoreIT { Query.Level.ALL_PROPERTIES, false); int count = 0; + List<Entity> list = new ArrayList<>(); while ( true ) { if (results.getEntities().size() == 0) { @@ -225,6 +237,7 @@ public class CollectionDeleteTest extends AbstractCoreIT { } lastEntityUUID = e.getUuid(); count++; + list.add(e); } results = em.getCollection(em.getApplicationRef(), collectionName, lastEntityUUID, expectedEntities, @@ -233,6 +246,16 @@ public class CollectionDeleteTest extends AbstractCoreIT { } logger.info("read {} total entities", count); + if (count != expectedEntities) { + logger.info("Expected {} did not match actual {}", expectedEntities, count); + if (count < 20) { + for (Entity e : list) { + Object key = e.getProperty("key2"); + logger.info("Entity key {} ceated {}", key, e.getCreated()); + } + } + } + assertEquals( "Did not get expected entities", expectedEntities, count ); return count; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java index a1ff84b..57962c0 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java @@ -51,7 +51,7 @@ import static org.junit.Assert.fail; @NotThreadSafe public class RebuildIndexTest extends AbstractCoreIT { - private static final Logger logger = LoggerFactory.getLogger( RebuildIndexTest.class ); + private static final Logger logger = LoggerFactory.getLogger(RebuildIndexTest.class); private static final MetricRegistry registry = new MetricRegistry(); @@ -70,183 +70,182 @@ public class RebuildIndexTest extends AbstractCoreIT { @After public void printReport() { - logger.debug( "Printing metrics report" ); + logger.debug("Printing metrics report"); } - @Test( timeout = 240000 ) + @Test(timeout = 240000) public void rebuildOneCollectionIndex() throws Exception { - logger.info( "Started rebuildOneCollectionIndex()" ); + logger.info("Started rebuildOneCollectionIndex()"); - String rand = RandomStringUtils.randomAlphanumeric( 5 ); - final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand ); + String rand = RandomStringUtils.randomAlphanumeric(5); + final UUID appId = setup.createApplication("org_" + rand, "app_" + rand); - final EntityManager em = setup.getEmf().getEntityManager( appId ); + final EntityManager em = setup.getEmf().getEntityManager(appId); - final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class ); + final ReIndexService reIndexService = setup.getInjector().getInstance(ReIndexService.class); // ----------------- create a bunch of entities Map<String, Object> entityMap = new HashMap<String, Object>() {{ - put( "key1", 1000 ); - put( "key2", 2000 ); - put( "key3", "Some value" ); + put("key1", 1000); + put("key2", 2000); + put("key3", "Some value"); }}; List<EntityRef> entityRefs = new ArrayList<EntityRef>(); int herderCount = 0; int shepardCount = 0; - for ( int i = 0; i < ENTITIES_TO_INDEX; i++ ) { + for (int i = 0; i < ENTITIES_TO_INDEX; i++) { final Entity entity; try { - entityMap.put( "key", i ); + entityMap.put("key", i); - if ( i % 2 == 0 ) { - entity = em.create( "catherder", entityMap ); + if (i % 2 == 0) { + entity = em.create("catherder", entityMap); herderCount++; - } - else { - entity = em.create( "catshepard", entityMap ); + } else { + entity = em.create("catshepard", entityMap); shepardCount++; } - } - catch ( Exception ex ) { - throw new RuntimeException( "Error creating entity", ex ); + } catch (Exception ex) { + throw new RuntimeException("Error creating entity", ex); } - entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) ); - if ( i % 10 == 0 ) { - logger.info( "Created {} entities", i ); + entityRefs.add(new SimpleEntityRef(entity.getType(), entity.getUuid())); + if (i % 10 == 0) { + logger.info("Created {} entities", i); } } - logger.info( "Created {} entities", ENTITIES_TO_INDEX ); - app.waitForQueueDrainAndRefreshIndex(5000); + logger.info("Created {} entities", ENTITIES_TO_INDEX); + app.waitForQueueDrainAndRefreshIndex(1000); // ----------------- test that we can read them, should work fine - logger.debug( "Read the data" ); - readData( em, "catherders", herderCount, 0 ); - readData( em, "catshepards", shepardCount, 0 ); + logger.debug("Read the data"); + retryReadData(em, "catherders", herderCount, 0, 10); + retryReadData(em, "catshepards", shepardCount, 0, 10); // ----------------- delete the system and application indexes - logger.debug( "Deleting apps" ); - deleteIndex( em.getApplicationId() ); + logger.debug("Deleting apps"); + deleteIndex(em.getApplicationId()); // ----------------- test that we can read them, should fail - logger.debug( "Reading data, should fail this time " ); + logger.debug("Reading data, should fail this time "); //should be no data - readData( em, "testTypes", 0, 0 ); + readData(em, "testTypes", 0, 0); // ----------------- rebuild index for catherders only - logger.debug( "Preparing to rebuild all indexes" ); + logger.debug("Preparing to rebuild all indexes"); final ReIndexRequestBuilder builder = - reIndexService.getBuilder().withApplicationId( em.getApplicationId() ).withCollection( "catherders" ); + reIndexService.getBuilder().withApplicationId(em.getApplicationId()).withCollection("catherders"); - ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder ); + ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder); - assertNotNull( status.getJobId(), "JobId is present" ); + assertNotNull(status.getJobId(), "JobId is present"); - logger.info( "Rebuilt index" ); + logger.info("Rebuilt index"); - waitForRebuild( status, reIndexService ); + waitForRebuild(status, reIndexService); - app.waitForQueueDrainAndRefreshIndex(15000); + //app.waitForQueueDrainAndRefreshIndex(15000); // ----------------- test that we can read the catherder collection and not the catshepard - readData( em, "catherders", herderCount, 0 ); - readData( em, "catshepards", 0, 0 ); + retryReadData(em, "catherders", herderCount, 0, 30); + retryReadData(em, "catshepards", 0, 0, 30); } - @Test( timeout = 240000 ) + @Test(timeout = 240000) public void rebuildIndex() throws Exception { - logger.info( "Started rebuildIndex()" ); + logger.info("Started rebuildIndex()"); - String rand = RandomStringUtils.randomAlphanumeric( 5 ); - final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand ); + String rand = RandomStringUtils.randomAlphanumeric(5); + final UUID appId = setup.createApplication("org_" + rand, "app_" + rand); - final EntityManager em = setup.getEmf().getEntityManager( appId ); + final EntityManager em = setup.getEmf().getEntityManager(appId); - final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class ); + final ReIndexService reIndexService = setup.getInjector().getInstance(ReIndexService.class); // ----------------- create a bunch of entities Map<String, Object> entityMap = new HashMap<String, Object>() {{ - put( "key1", 1000 ); - put( "key2", 2000 ); - put( "key3", "Some value" ); + put("key1", 1000); + put("key2", 2000); + put("key3", "Some value"); }}; Map<String, Object> cat1map = new HashMap<String, Object>() {{ - put( "name", "enzo" ); - put( "color", "orange" ); + put("name", "enzo"); + put("color", "orange"); }}; Map<String, Object> cat2map = new HashMap<String, Object>() {{ - put( "name", "marquee" ); - put( "color", "grey" ); + put("name", "marquee"); + put("color", "grey"); }}; Map<String, Object> cat3map = new HashMap<String, Object>() {{ - put( "name", "bertha" ); - put( "color", "tabby" ); + put("name", "bertha"); + put("color", "tabby"); }}; - Entity cat1 = em.create( "cat", cat1map ); - Entity cat2 = em.create( "cat", cat2map ); - Entity cat3 = em.create( "cat", cat3map ); + Entity cat1 = em.create("cat", cat1map); + Entity cat2 = em.create("cat", cat2map); + Entity cat3 = em.create("cat", cat3map); List<EntityRef> entityRefs = new ArrayList<>(); - for ( int i = 0; i < ENTITIES_TO_INDEX; i++ ) { + for (int i = 0; i < ENTITIES_TO_INDEX; i++) { final Entity entity; try { - entityMap.put( "key", i ); - entity = em.create( "testType", entityMap ); - - - em.createConnection( entity, "herds", cat1 ); - em.createConnection( entity, "herds", cat2 ); - em.createConnection( entity, "herds", cat3 ); - } - catch ( Exception ex ) { - throw new RuntimeException( "Error creating entity", ex ); + entityMap.put("key", i); + entity = em.create("testType", entityMap); + + em.createConnection(entity, "herds", cat1); + em.createConnection(entity, "herds", cat2); + em.createConnection(entity, "herds", cat3); + } catch (Exception ex) { + throw new RuntimeException("Error creating entity", ex); } - entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) ); - if ( i % 10 == 0 ) { - logger.info( "Created {} entities", i ); + entityRefs.add(new SimpleEntityRef(entity.getType(), entity.getUuid())); + if (i % 10 == 0) { + logger.info("Created {} entities", i); } } - logger.info( "Created {} entities", ENTITIES_TO_INDEX ); - app.waitForQueueDrainAndRefreshIndex(30000); + logger.info("Created {} entities", ENTITIES_TO_INDEX); + //app.waitForQueueDrainAndRefreshIndex(30000); // ----------------- test that we can read them, should work fine - logger.debug( "Read the data" ); + logger.debug("Read the data"); final String collectionName = "testtypes"; - readData( em, collectionName, ENTITIES_TO_INDEX, 3 ); + + retryReadData(em, collectionName, ENTITIES_TO_INDEX, 3, 20); + + readData(em, collectionName, ENTITIES_TO_INDEX, 3); // ----------------- delete the system and application indexes - logger.debug( "Deleting app index" ); + logger.debug("Deleting app index"); - deleteIndex( em.getApplicationId() ); + deleteIndex(em.getApplicationId()); app.waitForQueueDrainAndRefreshIndex(); @@ -257,64 +256,62 @@ public class RebuildIndexTest extends AbstractCoreIT { // ----------------- test that we can read them, should fail - logger.debug( "Reading data, should fail this time " ); - - readData( em, collectionName, 0, 0 ); + logger.debug("Reading data, should fail this time "); + readData(em, collectionName, 0, 0); // ----------------- rebuild index - logger.debug( "Preparing to rebuild all indexes" ); + logger.debug("Preparing to rebuild all indexes"); ; try { final ReIndexRequestBuilder builder = - reIndexService.getBuilder().withApplicationId( em.getApplicationId() ); + reIndexService.getBuilder().withApplicationId(em.getApplicationId()); - ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder ); + ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder); - assertNotNull( status.getJobId(), "JobId is present" ); + assertNotNull(status.getJobId(), "JobId is present"); - logger.info( "Rebuilt index, jobID={}", status.getJobId()); + logger.info("Rebuilt index, jobID={}", status.getJobId()); - waitForRebuild( status, reIndexService ); + waitForRebuild(status, reIndexService); - logger.info( "Rebuilt index" ); + logger.info("Rebuilt index"); - } - catch ( Exception ex ) { - logger.error( "Error rebuilding index", ex ); + } catch (Exception ex) { + logger.error("Error rebuilding index", ex); fail(); } // ----------------- test that we can read them app.waitForQueueDrainAndRefreshIndex(15000); - readData( em, collectionName, ENTITIES_TO_INDEX, 3 ); + readData(em, collectionName, ENTITIES_TO_INDEX, 3); } - @Test( timeout = 120000 ) + @Test(timeout = 120000) public void rebuildIndexGeo() throws Exception { - logger.info( "Started rebuildIndexGeo()" ); + logger.info("Started rebuildIndexGeo()"); - String rand = RandomStringUtils.randomAlphanumeric( 5 ); - final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand ); + String rand = RandomStringUtils.randomAlphanumeric(5); + final UUID appId = setup.createApplication("org_" + rand, "app_" + rand); - final EntityManager em = setup.getEmf().getEntityManager( appId ); + final EntityManager em = setup.getEmf().getEntityManager(appId); - final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class ); + final ReIndexService reIndexService = setup.getInjector().getInstance(ReIndexService.class); // ----------------- create a bunch of entities Map<String, Object> cat1map = new HashMap<String, Object>() {{ - put( "name", "enzo" ); - put( "color", "grey" ); + put("name", "enzo"); + put("color", "grey"); put("location", new LinkedHashMap<String, Object>() {{ put("latitude", -35.746369); put("longitude", 150.952183); @@ -323,44 +320,44 @@ public class RebuildIndexTest extends AbstractCoreIT { final double lat = -34.746369; final double lon = 152.952183; Map<String, Object> cat2map = new HashMap<String, Object>() {{ - put( "name", "marquee" ); - put( "color", "grey" ); + put("name", "marquee"); + put("color", "grey"); put("location", new LinkedHashMap<String, Object>() {{ put("latitude", lat); put("longitude", lon); }}); }}; Map<String, Object> cat3map = new HashMap<String, Object>() {{ - put( "name", "bertha" ); - put( "color", "grey" ); + put("name", "bertha"); + put("color", "grey"); put("location", new LinkedHashMap<String, Object>() {{ put("latitude", -33.746369); put("longitude", 150.952183); }}); }}; - Entity cat1 = em.create( "cat", cat1map ); - Entity cat2 = em.create( "cat", cat2map ); - Entity cat3 = em.create( "cat", cat3map ); + Entity cat1 = em.create("cat", cat1map); + Entity cat2 = em.create("cat", cat2map); + Entity cat3 = em.create("cat", cat3map); - logger.info( "Created {} entities", ENTITIES_TO_INDEX ); + logger.info("Created {} entities", ENTITIES_TO_INDEX); app.waitForQueueDrainAndRefreshIndex(5000); // ----------------- test that we can read them, should work fine - logger.debug( "Read the data" ); + logger.debug("Read the data"); final String collectionName = "cats"; - Query q = Query.fromQL( "select * where color='grey'" ).withLimit( 1000 ); - Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 3 ); - assertEquals(3,results.size()); + Query q = Query.fromQL("select * where color='grey'").withLimit(1000); + Results results = em.searchCollectionConsistent(em.getApplicationRef(), collectionName, q, 3); + assertEquals(3, results.size()); // ----------------- delete the system and application indexes - logger.debug( "Deleting app index" ); + logger.debug("Deleting app index"); - deleteIndex( em.getApplicationId() ); + deleteIndex(em.getApplicationId()); // ----------------- test that we can read them, should fail @@ -369,87 +366,86 @@ public class RebuildIndexTest extends AbstractCoreIT { // ----------------- test that we can read them, should fail - logger.debug( "Reading data, should fail this time " ); + logger.debug("Reading data, should fail this time "); - results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 0 ); - assertEquals(results.size(),0); + results = em.searchCollectionConsistent(em.getApplicationRef(), collectionName, q, 0); + assertEquals(results.size(), 0); // ----------------- rebuild index - logger.debug( "Preparing to rebuild all indexes" ); + logger.debug("Preparing to rebuild all indexes"); try { final ReIndexRequestBuilder builder = - reIndexService.getBuilder().withApplicationId( em.getApplicationId() ); + reIndexService.getBuilder().withApplicationId(em.getApplicationId()); - ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder ); + ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder); - assertNotNull( status.getJobId(), "JobId is present" ); + assertNotNull(status.getJobId(), "JobId is present"); - logger.info( "Rebuilt index" ); + logger.info("Rebuilt index"); - waitForRebuild( status, reIndexService ); + waitForRebuild(status, reIndexService); - logger.info( "Rebuilt index" ); + logger.info("Rebuilt index"); - } - catch ( Exception ex ) { - logger.error( "Error rebuilding index", ex ); + } catch (Exception ex) { + logger.error("Error rebuilding index", ex); fail(); } // ----------------- test that we can read them app.waitForQueueDrainAndRefreshIndex(5000); - results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 3 ); - assertEquals(results.size(),3); - q = Query.fromQL("select * where location within 100 of "+lat+", "+lon); - results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 1 ); - assertEquals(results.size(),1); + results = em.searchCollectionConsistent(em.getApplicationRef(), collectionName, q, 3); + assertEquals(results.size(), 3); + q = Query.fromQL("select * where location within 100 of " + lat + ", " + lon); + results = em.searchCollectionConsistent(em.getApplicationRef(), collectionName, q, 1); + assertEquals(results.size(), 1); } - @Test( timeout = 120000 ) + @Test(timeout = 120000) public void rebuildUpdatedSince() throws Exception { - logger.info( "Started rebuildUpdatedSince()" ); + logger.info("Started rebuildUpdatedSince()"); - String rand = RandomStringUtils.randomAlphanumeric( 5 ); - final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand ); + String rand = RandomStringUtils.randomAlphanumeric(5); + final UUID appId = setup.createApplication("org_" + rand, "app_" + rand); - final EntityManager em = setup.getEmf().getEntityManager( appId ); + final EntityManager em = setup.getEmf().getEntityManager(appId); - final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class ); + final ReIndexService reIndexService = setup.getInjector().getInstance(ReIndexService.class); // ----------------- create a bunch of entities Map<String, Object> entityData = new HashMap<String, Object>() {{ - put( "key1", 1000 ); + put("key1", 1000); }}; - final Entity firstEntity = em.create( "thing", entityData ); + final Entity firstEntity = em.create("thing", entityData); - final Entity secondEntity = em.create( "thing", entityData); + final Entity secondEntity = em.create("thing", entityData); app.waitForQueueDrainAndRefreshIndex(15000); // ----------------- test that we can read them, should work fine - logger.debug( "Read the data" ); + logger.debug("Read the data"); final String collectionName = "things"; - countEntities( em, collectionName, 2 ); + countEntities(em, collectionName, 2); // ----------------- delete the system and application indexes - logger.debug( "Deleting app index" ); + logger.debug("Deleting app index"); - deleteIndex( em.getApplicationId() ); + deleteIndex(em.getApplicationId()); // ----------------- test that we can read them, should fail @@ -462,14 +458,14 @@ public class RebuildIndexTest extends AbstractCoreIT { logger.debug("Reading data, should fail this time "); } - countEntities( em, collectionName, 0); + countEntities(em, collectionName, 0); // ----------------- rebuild index final long firstUpdatedTimestamp = firstEntity.getModified(); final long secondUpdatedTimestamp = secondEntity.getModified(); - assertTrue( "second should be updated after second", firstUpdatedTimestamp < secondUpdatedTimestamp ); + assertTrue("second should be updated after second", firstUpdatedTimestamp < secondUpdatedTimestamp); try { @@ -481,36 +477,35 @@ public class RebuildIndexTest extends AbstractCoreIT { //set our update timestamp final ReIndexRequestBuilder builder = - reIndexService.getBuilder().withApplicationId( em.getApplicationId() ).withStartTimestamp( - updatedTimestamp ); + reIndexService.getBuilder().withApplicationId(em.getApplicationId()).withStartTimestamp( + updatedTimestamp); - ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder ); + ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder); - assertNotNull( status.getJobId(), "JobId is present" ); + assertNotNull(status.getJobId(), "JobId is present"); - logger.info( "Rebuilt index" ); + logger.info("Rebuilt index"); - waitForRebuild( status, reIndexService ); + waitForRebuild(status, reIndexService); - logger.info( "Rebuilt index" ); + logger.info("Rebuilt index"); - } - catch ( Exception ex ) { - logger.error( "Error rebuilding index", ex ); + } catch (Exception ex) { + logger.error("Error rebuilding index", ex); fail(); } // ----------------- test that we can read them app.waitForQueueDrainAndRefreshIndex(5000); - countEntities( em, collectionName, 1 ); + countEntities(em, collectionName, 1); } /** * Wait for the rebuild to occur */ - private void waitForRebuild( final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService ) + private void waitForRebuild(final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService) throws InterruptedException, IllegalArgumentException { if (status != null) { logger.info("waitForRebuild: jobID={}", status.getJobId()); @@ -518,27 +513,26 @@ public class RebuildIndexTest extends AbstractCoreIT { logger.info("waitForRebuild: error, status = null"); throw new IllegalArgumentException("reindexStatus = null"); } - while ( true ) { + while (true) { try { - final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus( status.getJobId() ); + final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(status.getJobId()); if (updatedStatus == null) { logger.info("waitForRebuild: updated status is null"); } else { logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed()); - if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) { + if (updatedStatus.getStatus() == ReIndexService.Status.COMPLETE) { break; } } - } - catch ( IllegalArgumentException iae ) { + } catch (IllegalArgumentException iae) { //swallow. Thrown if our job can't be found. I.E hasn't updated yet } - Thread.sleep( 1000 ); + Thread.sleep(1000); } } @@ -546,22 +540,30 @@ public class RebuildIndexTest extends AbstractCoreIT { /** * Delete app index */ - private void deleteIndex( UUID appUuid ) { + private void deleteIndex(UUID appUuid) { - Injector injector = SpringResource.getInstance().getBean( Injector.class ); + Injector injector = SpringResource.getInstance().getBean(Injector.class); IndexLocationStrategyFactory indexLocationStrategyFactory = injector.getInstance(IndexLocationStrategyFactory.class); - EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class ); + EntityIndexFactory eif = injector.getInstance(EntityIndexFactory.class); - Id appId = new SimpleId( appUuid, Schema.TYPE_APPLICATION ); - ApplicationScope scope = new ApplicationScopeImpl( appId ); + Id appId = new SimpleId(appUuid, Schema.TYPE_APPLICATION); + ApplicationScope scope = new ApplicationScopeImpl(appId); EntityIndex ei = eif.createEntityIndex( indexLocationStrategyFactory.getIndexLocationStrategy(scope) ); - ei.deleteApplication().toBlocking().lastOrDefault( null ); + ei.deleteApplication().toBlocking().lastOrDefault(null); app.waitForQueueDrainAndRefreshIndex(); } + private int retryReadData(EntityManager em, String collectionName, int expectedEntities, int expectedConnections, int retry) throws Exception { + int count = readData(em, collectionName, expectedEntities, expectedConnections); + while (count != expectedEntities && --retry >=0) { + count = readData(em, collectionName, expectedEntities, expectedConnections); + } + assertEquals( "Did not get expected entities", expectedEntities, count ); + return count; + } private int readData( EntityManager em, String collectionName, int expectedEntities, int expectedConnections ) throws Exception { @@ -598,7 +600,6 @@ public class RebuildIndexTest extends AbstractCoreIT { } } - assertEquals( "Did not get expected entities", expectedEntities, count ); return count; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java index 9c09806..449eac9 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java @@ -28,6 +28,7 @@ import com.google.common.base.Preconditions; import org.apache.usergrid.corepersistence.index.CollectionDeleteRequestBuilder; import org.apache.usergrid.corepersistence.index.CollectionDeleteRequestBuilderImpl; import org.apache.usergrid.corepersistence.index.CollectionDeleteService; +import org.apache.usergrid.corepersistence.util.CpCollectionUtils; import org.apache.usergrid.persistence.index.utils.ConversionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +51,7 @@ import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; import java.util.HashMap; import java.util.Map; +import java.util.Properties; /** @@ -71,6 +73,34 @@ public class CollectionResource extends ServiceResource { } + /* + * Combine the passed settings with the settings already in the collection + */ + private void combineSettings(ServicePayload newSettings, UriInfo ui) throws Exception { + + + // Get the existing collection settings. + ApiResponse response = createApiResponse(); + response.setAction( "get" ); + response.setApplication( services.getApplication() ); + response.setParams( ui.getQueryParameters() ); + + executeServiceGetRequestForSettings( ui,response,ServiceAction.GET,null ); + + // now add the old settings to the new if they are not alreday in the new. + Object oldSettings = response.getData(); + if (oldSettings instanceof Map) { + Map<String,Object> oldProps = (Map) oldSettings; + Map<String,Object> newProps = newSettings.getProperties(); + for (String key : oldProps.keySet()) { + if (CpCollectionUtils.getValidSettings().contains(key) && !newProps.containsKey(key)) { + Object value = oldProps.get(key); + newSettings.setProperty(key, value); + } + } + } + } + /** * POST settings for a collection. * @@ -87,7 +117,9 @@ public class CollectionResource extends ServiceResource { @Context UriInfo ui, @PathParam("itemName") PathSegment itemName, String body, - @QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception { + @QueryParam("callback") @DefaultValue("callback") String callback, + @QueryParam("replace_all") @DefaultValue("true") String replace_all + ) throws Exception { if(logger.isTraceEnabled()){ logger.trace( "CollectionResource.executePostOnSettingsWithCollectionName" ); @@ -110,6 +142,10 @@ public class CollectionResource extends ServiceResource { ServicePayload payload = getPayload( json ); + if ("false".equals(replace_all)) { + combineSettings(payload, ui); + } + executeServicePostRequestForSettings( ui,response, ServiceAction.POST, payload ); return response; http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java index b8baa3d..eff9f2a 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java @@ -58,10 +58,10 @@ public class CollectionsResourceIT extends AbstractRestIT { private static final Logger log = LoggerFactory.getLogger( CollectionsResourceIT.class ); @Test - public void testValidSettings() throws IOException { + public void testPostValidSettings() throws IOException { String randomizer = RandomStringUtils.randomAlphanumeric(10); - String collectionName = "collection" + randomizer; + String collectionName = "collection_" + randomizer; //Create test collection with test entity that is full text indexed. Entity testEntity = new Entity(); @@ -91,12 +91,15 @@ public class CollectionsResourceIT extends AbstractRestIT { List list = (List) feildsValue; assertEquals(1, list.size() ); assertEquals( "one", list.get(0) ); - } + /** + * test that invalid settings are ignored + * @throws IOException + */ @Test - public void testInvalidSettings() throws IOException { + public void testPostInvalidSettings() throws IOException { String randomizer = RandomStringUtils.randomAlphanumeric(10); String collectionName = "collection" + randomizer; @@ -126,6 +129,78 @@ public class CollectionsResourceIT extends AbstractRestIT { } + + /** + * test that we can add new properties and modify existing ones. + * @throws IOException + */ + @Test + public void testAddNewSettings() throws IOException { + + String randomizer = RandomStringUtils.randomAlphanumeric(10); + String collectionName = "collection_" + randomizer; + + //Create test collection with test entity that is full text indexed. + Entity testEntity = new Entity(); + + testEntity.put( "one","value" ); + testEntity.put( "two","value" ); + app().collection( collectionName ).post( testEntity ); + + //field "fields" is required. + Entity payload = new Entity(); + payload.put( "fields", Collections.singletonList("one")); + + //Post index to the collection metadata + app().collection( collectionName ).collection( "_settings" ).post( payload ); + waitForQueueDrainAndRefreshIndex(); + + // Now add a new setting 'queueIndex' + + payload = new Entity(); + payload.put( "queueIndex", "direct"); + + Map<String, String> queryParams = Collections.singletonMap("replace_all","false"); + Entity settings = app().collection( collectionName ).collection( "_settings" ).post( payload , queryParams ); + waitForQueueDrainAndRefreshIndex(); + + // assert that both old and new are in the setting. + + Object queueIndexValue = settings.get("queueIndex"); + assertEquals( "direct", queueIndexValue); + + Object feildsValue = settings.get("fields"); + assertNotNull(feildsValue); + assertTrue(feildsValue instanceof List); + List list = (List) feildsValue; + assertEquals(1, list.size() ); + assertEquals( "one", list.get(0) ); + + // now replace the settings + + payload = new Entity(); + payload.put( "queueIndex", "async"); + payload.put( "fields", Collections.singletonList("two")); + + settings = app().collection( collectionName ).collection( "_settings" ).post( payload , queryParams ); + waitForQueueDrainAndRefreshIndex(); + + // assert that they have new values + + queueIndexValue = settings.get("queueIndex"); + assertEquals( "async", queueIndexValue); + + feildsValue = settings.get("fields"); + assertNotNull(feildsValue); + assertTrue(feildsValue instanceof List); + list = (List) feildsValue; + assertEquals(1, list.size() ); + assertEquals( "two", list.get(0) ); + + } + + + @Test public void testStaleEntries() throws IOException { http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java index af87ca5..66276bd 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java @@ -75,9 +75,21 @@ public class UserResourceIT extends AbstractRestIT { usersResource = this.app().collection("users"); userResource = this.app().collection("user"); + // make the queueindex direct to speed up the tests. + Entity payload = new Entity(); + payload.put( "queueIndex", "direct"); + + //Post index to the collection metadata + app().collection( "user" ).collection( "_settings" ).post( payload ); + app().collection( "users" ).collection( "_settings" ).post( payload ); + waitForQueueDrainAndRefreshIndex(); } + public void waitForQueueDrainAndRefreshIndex() { + waitForQueueDrainAndRefreshIndex(50); + } + @Test public void usernameQuery() throws IOException { String ql = "username = 'user*'"; @@ -398,7 +410,7 @@ public class UserResourceIT extends AbstractRestIT { @Test - public void clientNameQuery() { + public void clientNameQuery() throws Exception { String username = "username"; @@ -410,7 +422,18 @@ public class UserResourceIT extends AbstractRestIT { UUID createdId = entity.getUuid(); waitForQueueDrainAndRefreshIndex(); - Collection results = usersResource.get(new QueryParameters().setQuery(String.format("name = '%s'", name))); + + QueryParameters queryParameters = new QueryParameters().setQuery(String.format("name = '%s'", name)); + Collection results = usersResource.get(queryParameters); + + int retry = 30; + while (results.getResponse().getEntities().size() == 0 && --retry >= 0) { + Thread.sleep(1000); + results = usersResource.get(queryParameters); + logger.info("clientNameQuery results size {} ", results.getResponse().getEntities().size()); + } + assertEquals(1 , results.getResponse().getEntities().size()); + entity = new User(results.getResponse().getEntities().get(0)); assertEquals(createdId, entity.getUuid()); } @@ -1214,7 +1237,11 @@ public class UserResourceIT extends AbstractRestIT { ql = "uuid = " + userId; Collection response = this.app().collection("curts").get(new QueryParameters().setQuery(ql)); - + int retry = 30; + while (response.getResponse().getEntities().isEmpty() &&--retry >= 0) { + Thread.sleep(1000); + } + assertFalse(response.getResponse().getEntities().isEmpty()); assertEquals(response.getResponse().getEntities().get(0).get("uuid").toString(), userId.toString()); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java index d1c7ab4..9b1ad82 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java @@ -18,9 +18,7 @@ package org.apache.usergrid.rest.test.resource.endpoints; import java.io.IOException; import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.util.*; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; @@ -229,7 +227,7 @@ public class CollectionEndpoint extends NamedResource { } /** - * Post an entity to a collection. + * Post an entity to a collection with query params * * <pre> * app.collection("users").post(entity); @@ -237,15 +235,20 @@ public class CollectionEndpoint extends NamedResource { * </pre> */ public org.apache.usergrid.rest.test.resource.model.Entity post( - org.apache.usergrid.rest.test.resource.model.Entity payload){ + org.apache.usergrid.rest.test.resource.model.Entity payload, Map<String,String> queryParams){ String acceptHeader = MediaType.APPLICATION_JSON; if (this.acceptHeaders.size() > 0) { acceptHeader = StringUtils.join(this.acceptHeaders, ','); } + WebTarget target = getTarget(true); + for (String key : queryParams.keySet()) { + target = target.queryParam(key, queryParams.get(key)); + } + // use string type so we can log actual response from server - String responseString = getTarget( true ) + String responseString = target .request() .accept(acceptHeader) .post( javax.ws.rs.client.Entity.json( payload ), String.class); @@ -266,6 +269,21 @@ public class CollectionEndpoint extends NamedResource { return new org.apache.usergrid.rest.test.resource.model.Entity(response); } + + /** + * Post an entity to a collection. + * + * <pre> + * app.collection("users").post(entity); + * POST /users {"color","red"} + * </pre> + */ + public org.apache.usergrid.rest.test.resource.model.Entity post( + org.apache.usergrid.rest.test.resource.model.Entity payload){ + + return post(payload, Collections.emptyMap()); + } + public org.apache.usergrid.rest.test.resource.model.Entity post() { String acceptHeader = MediaType.APPLICATION_JSON; http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java index 9b3dfcd..bab3812 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java @@ -49,15 +49,17 @@ public abstract class AbstractServiceNotificationIT extends AbstractServiceIT { protected Notification notificationWaitForComplete(Notification notification) throws Exception { - long timeout = System.currentTimeMillis() + 120000; - while (System.currentTimeMillis() < timeout) { - app.waitForQueueDrainAndRefreshIndex(1000); + + int retry = 18; // 3 mins 18 * 10 seconds + while (-- retry > 0) { + logger.info("notificationWaitForComplete {} retry {}", notification.getUuid(), retry); + app.waitForQueueDrainAndRefreshIndex(10000); notification = app.getEntityManager().get(notification.getUuid(), Notification.class); if (notification.getFinished() != null) { return notification; } } - fail("Notification failed to complete"); + fail("Notification failed to complete error message " + notification.getErrorMessage()); return null; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java index 4fb9add..09ecc59 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java @@ -323,13 +323,15 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT { // delay until the scheduler has time to run - logger.info("Sleeping while the scheduler does its work"); - Thread.sleep(5000); - - - notification = app.getEntityManager().get(e.getUuid(), Notification.class); - - assertEquals(Notification.State.FINISHED, notification.getState()); + Notification.State notificationState = Notification.State.SCHEDULED; + int retry = 60; + do { + logger.info("Sleeping while the scheduler does its work " + retry); + Thread.sleep(1000); + notificationState = app.getEntityManager().get(e.getUuid(), Notification.class).getState(); + } while (Notification.State.FINISHED != notificationState && --retry >= 0 ); + + assertEquals(Notification.State.FINISHED, notificationState); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java index ecc5443..d403d5b 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java @@ -632,10 +632,17 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT { // receipts are created and queried, wait a bit longer for this to happen as indexing - app.waitForQueueDrainAndRefreshIndex(5000); + app.waitForQueueDrainAndRefreshIndex(2000); // get the receipts entity IDs List<EntityRef> receipts = getNotificationReceipts(notification); + + int retry = 30; + while (receipts.size() == 0 && --retry >=0 ) { + app.waitForQueueDrainAndRefreshIndex(1000); + receipts = getNotificationReceipts(notification); + } + assertEquals(1, receipts.size()); // Validate the error is the correct type InvalidRegistration
