usergrid git commit: remove excess interfaces

2015-10-19 Thread sfeldman 
Repository: usergrid
Updated Branches:
  refs/heads/remove-inmemory-event-service e5e9d021b -> efffe474b


remove excess interfaces


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/efffe474
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/efffe474
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/efffe474

Branch: refs/heads/remove-inmemory-event-service
Commit: efffe474bdca1223222f955204301f604dea40b8
Parents: e5e9d02
Author: Shawn Feldman 
Authored: Mon Oct 19 11:38:25 2015 -0600
Committer: Shawn Feldman 
Committed: Mon Oct 19 11:38:25 2015 -0600

--
 .../corepersistence/CpEntityManager.java|  7 +-
 .../corepersistence/CpEntityManagerFactory.java |  3 +-
 .../asyncevents/AsyncIndexProvider.java |  7 +-
 .../usergrid/persistence/EntityManager.java |  5 +-
 .../persistence/EntityManagerFactory.java   |  4 +-
 .../corepersistence/StaleIndexCleanupTest.java  |  5 +-
 .../persistence/query/IteratingQueryIT.java |  1 +
 .../usergrid/persistence/index/EntityIndex.java | 20 +++-
 .../persistence/index/IndexRefreshCommand.java  | 50 --
 .../persistence/index/guice/IndexModule.java|  1 -
 .../index/impl/EsEntityIndexFactoryImpl.java|  6 +-
 .../index/impl/EsEntityIndexImpl.java   | 38 ++--
 .../index/impl/IndexRefreshCommandImpl.java | 97 ---
 .../persistence/index/impl/EntityIndexTest.java |  2 +-
 .../persistence/queue/DefaultQueueManager.java  | 98 
 .../persistence/queue/LocalQueueManager.java| 98 
 .../queue/impl/QueueManagerFactoryImpl.java |  4 +-
 .../apns/NotificationsServiceIT.java|  4 +-
 .../gcm/NotificationsServiceIT.java |  2 -
 19 files changed, 170 insertions(+), 282 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/efffe474/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index addd6ef..4b32389 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -40,7 +40,6 @@ import 
org.apache.usergrid.corepersistence.service.CollectionService;
 import org.apache.usergrid.corepersistence.service.ConnectionService;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.IndexLocationStrategy;
-import org.apache.usergrid.persistence.index.IndexRefreshCommand;
 import org.apache.usergrid.persistence.index.utils.*;
 import org.apache.usergrid.utils.*;
 import org.apache.usergrid.utils.ClassUtils;
@@ -2877,7 +2876,7 @@ public class CpEntityManager implements EntityManager {
 /**
  * TODO, these 3 methods are super janky.  During refactoring we should 
clean this model up
  */
-public IndexRefreshCommand.IndexRefreshCommandInfo refreshIndex() {
+public EntityIndex.IndexRefreshCommandInfo refreshIndex() {
 try {
 long start = System.currentTimeMillis();
 // refresh special indexes without calling EntityManager refresh 
because stack overflow
@@ -2885,7 +2884,7 @@ public class CpEntityManager implements EntityManager {
 map.put("some prop", "test");
 boolean hasFinished = false;
 Entity refreshEntity = create("refresh", map);
-IndexRefreshCommand.IndexRefreshCommandInfo indexRefreshCommandInfo
+EntityIndex.IndexRefreshCommandInfo indexRefreshCommandInfo
 = 
managerCache.getEntityIndex(applicationScope).refreshAsync().toBlocking().first();
 try {
 for (int i = 0; i < 10; i++) {
@@ -2899,6 +2898,7 @@ public class CpEntityManager implements EntityManager {
 break;
 }
 Thread.sleep(200);
+
 indexRefreshCommandInfo
 = 
managerCache.getEntityIndex(applicationScope).refreshAsync().toBlocking().first();
 }
@@ -2908,6 +2908,7 @@ public class CpEntityManager implements EntityManager {
 }finally {
 delete(refreshEntity);
 }
+Thread.sleep(200);
 
 return indexRefreshCommandInfo;
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/efffe474/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java

usergrid git commit: Adds comment

2015-10-19 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1048 19d30eafc -> 4013f17e4


Adds comment


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4013f17e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4013f17e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4013f17e

Branch: refs/heads/USERGRID-1048
Commit: 4013f17e434a5eb05f4851d28dba6a3e0bfe8fc8
Parents: 19d30ea
Author: Todd Nine 
Authored: Mon Oct 19 14:10:51 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 19 14:10:51 2015 -0600

--
 .../corepersistence/asyncevents/AmazonAsyncEventService.java | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/4013f17e/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 67d0dab..ee9054b 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -536,6 +536,10 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 indexOperationMessage = ObjectJsonSerializer.INSTANCE.fromString( 
message, IndexOperationMessage.class );
 }
 
+
+//NOTE that we intentionally do NOT delete from the map.  We can't 
know when all regions have consumed the message
+//so we'll let compaction on column expiration handle deletion
+
 //read the value from the string
 
 Preconditions.checkNotNull( indexOperationMessage, 
"indexOperationMessage cannot be null" );

usergrid git commit: Makes consistency configurable

2015-10-19 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1048 0326629a2 -> 19d30eafc


Makes consistency configurable


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/19d30eaf
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/19d30eaf
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/19d30eaf

Branch: refs/heads/USERGRID-1048
Commit: 19d30eafc77095ee74ae126f9d0a849e997b6ad7
Parents: 0326629
Author: Todd Nine 
Authored: Mon Oct 19 13:59:08 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 19 13:59:08 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java|  4 +---
 .../map/impl/MapSerializationImpl.java  | 21 ++--
 2 files changed, 12 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/19d30eaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 6b2eb45..67d0dab 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -29,9 +29,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.google.common.base.Optional;
-import org.apache.usergrid.persistence.index.impl.IndexProducer;
-import org.apache.usergrid.persistence.queue.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -68,6 +65,7 @@ import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueFig;
 import org.apache.usergrid.persistence.queue.QueueManager;
 import org.apache.usergrid.persistence.queue.QueueManagerFactory;
 import org.apache.usergrid.persistence.queue.QueueMessage;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/19d30eaf/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
--
diff --git 
a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
 
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
index 1aa3229..ffe10c9 100644
--- 
a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
+++ 
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
@@ -33,6 +33,8 @@ import org.apache.cassandra.db.marshal.UTF8Type;
 
 import org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKey;
 import 
org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
 import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
 import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
 import 
org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
@@ -105,15 +107,9 @@ public class MapSerializationImpl implements 
MapSerialization {
 /**
  * How to funnel keys for buckets
  */
-private static final Funnel MAP_KEY_FUNNEL = new Funnel() {
+private static final Funnel MAP_KEY_FUNNEL = ( key, into ) -> 
into.putString( key, StringHashUtils.UTF8 );
 
 
-@Override
-public void funnel( final String key, final PrimitiveSink into ) {
-into.putString( key, StringHashUtils.UTF8 );
-}
-};
-
 /**
  * Locator to get us all buckets
  */
@@ -121,10 +117,14 @@ public class MapSerializationImpl implements 
MapSerialization {
 new ExpandingShardLocator<>( MAP_KEY_FUNNEL, NUM_BUCKETS );
 
 private final Keyspace keyspace;
+private final CassandraConfig cassandraConfig;
 
 
 @Inject
-public MapSerializationImpl( final Keyspace keyspace ) {this.keyspace = 
keyspace;}
+public MapSerializationImpl( final Keyspace keyspace, final 
CassandraConfig cassandraConfig ) {
+this.keyspace = keyspace;
+this.cassandraConfig = cassandraConfig;
+

[1/8] usergrid git commit: Adds strong consistency read to maps. Persists ES batches into Cassandra for multi region execution.

2015-10-19 Thread sfeldman 
Repository: usergrid
Updated Branches:
  refs/heads/remove-inmemory-event-service 00175cc61 -> 5eed63d43


Adds strong consistency read to maps.  Persists ES batches into Cassandra for 
multi region execution.

A bug in wiring JSON to SQS still exists, it's incorrectly escaping some 
message subtypes.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/94a90781
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/94a90781
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/94a90781

Branch: refs/heads/remove-inmemory-event-service
Commit: 94a9078125fc32d755e33e562f8e8fd8624641c1
Parents: 2b22c61
Author: Todd Nine 
Authored: Fri Oct 16 18:02:44 2015 -0600
Committer: Todd Nine 
Committed: Fri Oct 16 18:02:44 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java| 186 +++---
 .../asyncevents/AsyncEventService.java  |   1 +
 .../asyncevents/AsyncIndexProvider.java |  22 +-
 .../asyncevents/model/AsyncEvent.java   |   3 +-
 .../model/ElasticsearchIndexEvent.java  |  50 
 .../index/IndexProcessorFig.java|   8 +
 .../util/ObjectJsonSerializer.java  |  74 ++
 .../index/AmazonAsyncEventServiceTest.java  |   6 +-
 .../index/AsyncIndexServiceTest.java|   2 +-
 .../usergrid/persistence/map/MapManager.java|  25 +-
 .../persistence/map/impl/MapManagerImpl.java|   6 +
 .../persistence/map/impl/MapSerialization.java  |  27 +-
 .../map/impl/MapSerializationImpl.java  | 248 ++-
 .../index/impl/DeIndexOperation.java|   4 +
 .../persistence/index/impl/IndexOperation.java  |   4 +
 .../index/impl/IndexOperationMessage.java   |   5 +
 .../persistence/queue/DefaultQueueManager.java  |  12 +-
 .../persistence/queue/QueueManager.java |   8 +-
 .../queue/impl/SNSQueueManagerImpl.java | 188 ++
 .../queue/impl/SQSQueueManagerImpl.java |  28 ++-
 .../services/queues/ImportQueueManager.java |   9 +-
 21 files changed, 666 insertions(+), 250 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/94a90781/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 95126c6..c9f0953 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -21,13 +21,20 @@ package org.apache.usergrid.corepersistence.asyncevents;
 
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.google.common.base.Optional;
+
+import 
org.apache.usergrid.corepersistence.asyncevents.model.ElasticsearchIndexEvent;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.corepersistence.util.ObjectJsonSerializer;
+import org.apache.usergrid.exception.NotImplementedException;
 import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,8 +61,13 @@ import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexLocationStrategy;
 import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 import org.apache.usergrid.persistence.queue.QueueManager;
 import org.apache.usergrid.persistence.queue.QueueManagerFactory;
 import org.apache.usergrid.persistence.queue.QueueMessage;
@@ -82,12 +94,13 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 
 private static final Logger logger = 
LoggerFactory.getLogger(AmazonAsyncEventService.class);
 
+private static final ObjectJsonSerializer OBJECT_JSON_SERIALIZER = new 
ObjectJsonSerializer(  );
+

[3/8] usergrid git commit: Merge branch 'refs/heads/2.1-release' into USERGRID-1048

2015-10-19 Thread sfeldman 
Merge branch 'refs/heads/2.1-release' into USERGRID-1048


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3e155852
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3e155852
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3e155852

Branch: refs/heads/remove-inmemory-event-service
Commit: 3e1558524728c96834cfd66d9e53e3f1b6a7d3d6
Parents: 04a3f47 a09485a
Author: Todd Nine 
Authored: Mon Oct 19 13:30:04 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 19 13:30:04 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java| 149 ++-
 .../asyncevents/AsyncIndexProvider.java |  26 ++--
 .../asyncevents/model/AsyncEvent.java   |  14 +-
 .../asyncevents/model/EdgeDeleteEvent.java  |   6 +-
 .../asyncevents/model/EdgeIndexEvent.java   |   9 +-
 .../asyncevents/model/EntityDeleteEvent.java|   8 +-
 .../asyncevents/model/EntityIndexEvent.java |   6 +-
 .../model/InitializeApplicationIndexEvent.java  |   4 +-
 .../index/AmazonAsyncEventServiceTest.java  |   6 +-
 .../cache/CachedEntityCollectionManager.java| 147 --
 .../EntityCollectionManagerFactoryImpl.java |   6 -
 .../usergrid/persistence/queue/QueueFig.java|   2 +-
 .../queue/impl/SNSQueueManagerImpl.java |   8 +-
 13 files changed, 135 insertions(+), 256 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3e155852/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --cc 
stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index d319ac8,c198674..f8ef5e7
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@@ -97,7 -84,9 +100,8 @@@ public class AmazonAsyncEventService im
  public static final String QUEUE_NAME = "index"; //keep this short as AWS 
limits queue name size to 80 chars
  
  private final QueueManager queue;
 -private final QueueScope queueScope;
  private final IndexProcessorFig indexProcessorFig;
+ private final QueueFig queueFig;
  private final IndexProducer indexProducer;
  private final EntityCollectionManagerFactory 
entityCollectionManagerFactory;
  private final IndexLocationStrategyFactory indexLocationStrategyFactory;
@@@ -125,33 -113,28 +129,35 @@@
  
  
  @Inject
 -public AmazonAsyncEventService(final QueueManagerFactory 
queueManagerFactory,
 -   final IndexProcessorFig indexProcessorFig,
 -   final IndexProducer indexProducer,
 -   final MetricsFactory metricsFactory,
 -   final EntityCollectionManagerFactory 
entityCollectionManagerFactory,
 -   final IndexLocationStrategyFactory 
indexLocationStrategyFactory,
 -   final EntityIndexFactory 
entityIndexFactory,
 -   final EventBuilder eventBuilder,
 -   final RxTaskScheduler rxTaskScheduler,
 -   QueueFig queueFig) {
 +public AmazonAsyncEventService( final QueueManagerFactory 
queueManagerFactory,
 +final IndexProcessorFig indexProcessorFig,
 +final IndexProducer indexProducer,
 +final MetricsFactory metricsFactory,
 +final EntityCollectionManagerFactory 
entityCollectionManagerFactory,
 +final IndexLocationStrategyFactory 
indexLocationStrategyFactory,
 +final EntityIndexFactory 
entityIndexFactory,
 +final EventBuilder eventBuilder,
 +final MapManagerFactory mapManagerFactory,
++final QueueFig queueFig,
 +final RxTaskScheduler rxTaskScheduler ) {
  this.indexProducer = indexProducer;
  
  this.entityCollectionManagerFactory = entityCollectionManagerFactory;
  this.indexLocationStrategyFactory = indexLocationStrategyFactory;
  this.entityIndexFactory = entityIndexFactory;
  this.eventBuilder = eventBuilder;
 +
 +final MapScope mapScope = new MapScopeImpl( 
CpNamingUtils.getManagementApplicationId(),  "indexEvents");
 +
 +

[5/8] usergrid git commit: Makes consistency configurable

2015-10-19 Thread sfeldman 
Makes consistency configurable


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/19d30eaf
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/19d30eaf
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/19d30eaf

Branch: refs/heads/remove-inmemory-event-service
Commit: 19d30eafc77095ee74ae126f9d0a849e997b6ad7
Parents: 0326629
Author: Todd Nine 
Authored: Mon Oct 19 13:59:08 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 19 13:59:08 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java|  4 +---
 .../map/impl/MapSerializationImpl.java  | 21 ++--
 2 files changed, 12 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/19d30eaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 6b2eb45..67d0dab 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -29,9 +29,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.google.common.base.Optional;
-import org.apache.usergrid.persistence.index.impl.IndexProducer;
-import org.apache.usergrid.persistence.queue.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -68,6 +65,7 @@ import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueFig;
 import org.apache.usergrid.persistence.queue.QueueManager;
 import org.apache.usergrid.persistence.queue.QueueManagerFactory;
 import org.apache.usergrid.persistence.queue.QueueMessage;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/19d30eaf/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
--
diff --git 
a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
 
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
index 1aa3229..ffe10c9 100644
--- 
a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
+++ 
b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
@@ -33,6 +33,8 @@ import org.apache.cassandra.db.marshal.UTF8Type;
 
 import org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKey;
 import 
org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
 import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
 import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
 import 
org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
@@ -105,15 +107,9 @@ public class MapSerializationImpl implements 
MapSerialization {
 /**
  * How to funnel keys for buckets
  */
-private static final Funnel MAP_KEY_FUNNEL = new Funnel() {
+private static final Funnel MAP_KEY_FUNNEL = ( key, into ) -> 
into.putString( key, StringHashUtils.UTF8 );
 
 
-@Override
-public void funnel( final String key, final PrimitiveSink into ) {
-into.putString( key, StringHashUtils.UTF8 );
-}
-};
-
 /**
  * Locator to get us all buckets
  */
@@ -121,10 +117,14 @@ public class MapSerializationImpl implements 
MapSerialization {
 new ExpandingShardLocator<>( MAP_KEY_FUNNEL, NUM_BUCKETS );
 
 private final Keyspace keyspace;
+private final CassandraConfig cassandraConfig;
 
 
 @Inject
-public MapSerializationImpl( final Keyspace keyspace ) {this.keyspace = 
keyspace;}
+public MapSerializationImpl( final Keyspace keyspace, final 
CassandraConfig cassandraConfig ) {
+this.keyspace = keyspace;
+this.cassandraConfig = cassandraConfig;
+}
 
 
 @Override
@@ -387,7 +387,7 @@ public class

[7/8] usergrid git commit: merge

2015-10-19 Thread sfeldman 
merge


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/68befd2f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/68befd2f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/68befd2f

Branch: refs/heads/remove-inmemory-event-service
Commit: 68befd2f04d03dbd3e9370af89c257a8a28112a1
Parents: 00175cc
Author: Shawn Feldman 
Authored: Mon Oct 19 15:13:34 2015 -0600
Committer: Shawn Feldman 
Committed: Mon Oct 19 15:13:34 2015 -0600

--
 .../index/IndexProcessorFig.java| 36 ++--
 1 file changed, 11 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/68befd2f/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 113b539..8a8c8a5 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -49,15 +49,15 @@ public interface IndexProcessorFig extends GuicyFig {
  * Set the amount of time to wait when Elasticsearch rejects a requests 
before
  * retrying.  This provides simple back pressure. (in milliseconds)
  */
-@Default( "1000" )
-@Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
+@Default("1000")
+@Key(FAILURE_REJECTED_RETRY_WAIT_TIME)
 long getFailureRetryTime();
 
 /**
  * Set the read timeout for processing messages in the queue. (in 
milliseconds)
  */
-@Default( "1" )
-@Key( INDEX_QUEUE_READ_TIMEOUT )
+@Default("1")
+@Key(INDEX_QUEUE_READ_TIMEOUT)
 int getIndexQueueTimeout();
 
 /**
@@ -65,15 +65,15 @@ public interface IndexProcessorFig extends GuicyFig {
  * Received messages will remain 'in flight' until they are ack'd(deleted) 
or this timeout occurs.
  * If the timeout occurs, the messages will become visible again for 
re-processing.
  */
-@Default( "5000" ) // 5 seconds
-@Key( INDEX_QUEUE_VISIBILITY_TIMEOUT )
+@Default("5000") // 5 seconds
+@Key(INDEX_QUEUE_VISIBILITY_TIMEOUT)
 int getIndexQueueVisibilityTimeout();
 
 /**
  * The number of worker threads used to read index write requests from the 
queue.
  */
-@Default( "16" )
-@Key( ELASTICSEARCH_WORKER_COUNT )
+@Default("16")
+@Key(ELASTICSEARCH_WORKER_COUNT)
 int getWorkerCount();
 
 /**
@@ -81,26 +81,12 @@ public interface IndexProcessorFig extends GuicyFig {
  * Valid values: TEST, LOCAL, SQS, SNS
  * NOTE: SQS and SNS equate to the same implementation of Amazon queue 
services.
  */
-@Default( "LOCAL" )
-@Key( ELASTICSEARCH_QUEUE_IMPL )
+@Default("LOCAL")
+@Key(ELASTICSEARCH_QUEUE_IMPL)
 String getQueueImplementation();
 
 @Default("1000")
-@Key("elasticsearch.reindex.flush.interval")
-int getUpdateInterval();
-
-@Default("100")
-@Key("elasticsearch.buffer.time_ms")
-int getBufferTime();
-
-@Default("1000")
-@Key( REINDEX_BUFFER_SIZE )
+@Key(REINDEX_BUFFER_SIZE)
 int getReindexBufferSize();
 
-/**
- * Flag to resolve the LOCAL queue implementation service synchronously.
- */
-@Default("false")
-@Key("elasticsearch.queue_impl.resolution")
-boolean resolveSynchronously();
 }

usergrid git commit: Fixes empty payload notification issue.

2015-10-19 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1048 3e1558524 -> 0326629a2


Fixes empty payload notification issue.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0326629a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0326629a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0326629a

Branch: refs/heads/USERGRID-1048
Commit: 0326629a24cec3bd44d91810b4b8f0516c69c9b8
Parents: 3e15585
Author: Todd Nine 
Authored: Mon Oct 19 13:53:30 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 19 13:53:30 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java| 55 
 .../asyncevents/AsyncIndexProvider.java |  4 +-
 .../index/AmazonAsyncEventServiceTest.java  |  2 +-
 3 files changed, 36 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/0326629a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index f8ef5e7..6b2eb45 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -89,6 +89,21 @@ import rx.Subscription;
 import rx.schedulers.Schedulers;
 
 
+/**
+ * TODO, this whole class is becoming a nightmare.  We need to remove all 
consume from this class and refactor it into the following manner.
+ *
+ * 1. Produce.  Keep the code in the handle as is
+ * 2. Consume:  Move the code into a refactored system
+ * 2.1 A central dispatcher
+ * 2.2 An interface that produces an observable of type BatchOperation.  Any 
handler will be refactored into it's own
+ *  impl that will then emit a stream of batch operations to perform
+ * 2.3 The central dispatcher will then subscribe to these events and merge 
them.  Handing them off to a batch handler
+ * 2.4 The batch handler will roll up the operations into a batch size, and 
then queue them
+ * 2.5 The receive batch handler will execute the batch operations
+ *
+ * TODO determine how we error handle?
+ *
+ */
 @Singleton
 public class AmazonAsyncEventService implements AsyncEventService {
 
@@ -360,7 +375,8 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 public void queueInitializeApplicationIndex( final ApplicationScope 
applicationScope) {
 IndexLocationStrategy indexLocationStrategy = 
indexLocationStrategyFactory.getIndexLocationStrategy(
 applicationScope );
-offerTopic(new 
InitializeApplicationIndexEvent(queueFig.getPrimaryRegion(), new 
ReplicatedIndexLocationStrategy(indexLocationStrategy)));
+offerTopic( new InitializeApplicationIndexEvent( 
queueFig.getPrimaryRegion(),
+new ReplicatedIndexLocationStrategy( indexLocationStrategy ) ) );
 }
 
 
@@ -503,35 +519,29 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 
 final String message = esMapPersistence.getString( 
messageId.toString() );
 
-String highConsistency = null;
+final IndexOperationMessage indexOperationMessage;
 
 if(message == null){
 logger.error( "Received message with id {} to process, unable to 
find it, reading with higher consistency level" );
 
-highConsistency =  esMapPersistence.getStringHighConsistency( 
messageId.toString() );
-
-}
+final String highConsistency =  
esMapPersistence.getStringHighConsistency( messageId.toString() );
 
-//read the value from the string
+if(highConsistency == null){
+logger.error( "Unable to find the ES batch with id {} to 
process at a higher consistency level" );
 
-final IndexOperationMessage indexOperationMessage;
+throw new RuntimeException( "Unable to find the ES batch to 
process with message id " + messageId );
+}
 
-//our original local read has it, parse it.
-if(message != null){
- indexOperationMessage = ObjectJsonSerializer.INSTANCE.fromString( 
message, IndexOperationMessage.class );
-}
-//we tried to read it at a higher consistency level and it works
-else if (highConsistency != null){
 indexOperationMessage = ObjectJsonSerializer.INSTANCE.fromString( 
highConsistency, IndexOperationMessage.class );
-}
 
-//we couldn't find it, bail
-else{
-

usergrid git commit: Add sourceRegion to SQS event message based on queue primary region. Remove cached entity manager (unused).

2015-10-19 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/2.1-release 2b22c610d -> a09485a3a


Add sourceRegion to SQS event message based on queue primary region.  Remove 
cached entity manager (unused).


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a09485a3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a09485a3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a09485a3

Branch: refs/heads/2.1-release
Commit: a09485a3a5ac8b4217b294f4754ea8a70a7ec447
Parents: 2b22c61
Author: Michael Russo 
Authored: Fri Oct 16 13:12:42 2015 -0700
Committer: Michael Russo 
Committed: Fri Oct 16 13:12:42 2015 -0700

--
 .../asyncevents/AmazonAsyncEventService.java|  40 ++---
 .../asyncevents/AsyncIndexProvider.java |  10 +-
 .../asyncevents/model/AsyncEvent.java   |  14 +-
 .../asyncevents/model/EdgeDeleteEvent.java  |   6 +-
 .../asyncevents/model/EdgeIndexEvent.java   |   9 +-
 .../asyncevents/model/EntityDeleteEvent.java|   8 +-
 .../asyncevents/model/EntityIndexEvent.java |   6 +-
 .../model/InitializeApplicationIndexEvent.java  |   4 +-
 .../index/AmazonAsyncEventServiceTest.java  |   6 +-
 .../cache/CachedEntityCollectionManager.java| 147 ---
 .../EntityCollectionManagerFactoryImpl.java |   6 -
 .../usergrid/persistence/queue/QueueFig.java|   2 +-
 .../queue/impl/SNSQueueManagerImpl.java |   6 +-
 .../queue/impl/SQSQueueManagerImpl.java |   2 +-
 14 files changed, 64 insertions(+), 202 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a09485a3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 95126c6..c198674 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -29,6 +29,7 @@ import java.util.stream.Stream;
 
 import com.google.common.base.Optional;
 import org.apache.usergrid.persistence.index.impl.IndexProducer;
+import org.apache.usergrid.persistence.queue.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -56,10 +57,6 @@ import 
org.apache.usergrid.persistence.index.IndexLocationStrategy;
 import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
 import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
 
 import com.codahale.metrics.Counter;
@@ -89,6 +86,7 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 private final QueueManager queue;
 private final QueueScope queueScope;
 private final IndexProcessorFig indexProcessorFig;
+private final QueueFig queueFig;
 private final IndexProducer indexProducer;
 private final EntityCollectionManagerFactory 
entityCollectionManagerFactory;
 private final IndexLocationStrategyFactory indexLocationStrategyFactory;
@@ -115,15 +113,16 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 
 
 @Inject
-public AmazonAsyncEventService( final QueueManagerFactory 
queueManagerFactory,
-final IndexProcessorFig indexProcessorFig,
-final IndexProducer indexProducer,
-final MetricsFactory metricsFactory,
-final EntityCollectionManagerFactory 
entityCollectionManagerFactory,
-final IndexLocationStrategyFactory 
indexLocationStrategyFactory,
-final EntityIndexFactory 
entityIndexFactory,
-final EventBuilder eventBuilder,
-final RxTaskScheduler rxTaskScheduler ) {
+public AmazonAsyncEventService(final QueueManagerFactory 
queueManagerFactory,
+   final IndexProcessorFig indexProcessorFig,
+   final IndexProducer indexProducer,
+   final

[1/2] usergrid git commit: Add sourceRegion to SQS event message based on queue primary region. Remove cached entity manager (unused).

2015-10-19 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1048 04a3f47bd -> 3e1558524


Add sourceRegion to SQS event message based on queue primary region.  Remove 
cached entity manager (unused).


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a09485a3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a09485a3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a09485a3

Branch: refs/heads/USERGRID-1048
Commit: a09485a3a5ac8b4217b294f4754ea8a70a7ec447
Parents: 2b22c61
Author: Michael Russo 
Authored: Fri Oct 16 13:12:42 2015 -0700
Committer: Michael Russo 
Committed: Fri Oct 16 13:12:42 2015 -0700

--
 .../asyncevents/AmazonAsyncEventService.java|  40 ++---
 .../asyncevents/AsyncIndexProvider.java |  10 +-
 .../asyncevents/model/AsyncEvent.java   |  14 +-
 .../asyncevents/model/EdgeDeleteEvent.java  |   6 +-
 .../asyncevents/model/EdgeIndexEvent.java   |   9 +-
 .../asyncevents/model/EntityDeleteEvent.java|   8 +-
 .../asyncevents/model/EntityIndexEvent.java |   6 +-
 .../model/InitializeApplicationIndexEvent.java  |   4 +-
 .../index/AmazonAsyncEventServiceTest.java  |   6 +-
 .../cache/CachedEntityCollectionManager.java| 147 ---
 .../EntityCollectionManagerFactoryImpl.java |   6 -
 .../usergrid/persistence/queue/QueueFig.java|   2 +-
 .../queue/impl/SNSQueueManagerImpl.java |   6 +-
 .../queue/impl/SQSQueueManagerImpl.java |   2 +-
 14 files changed, 64 insertions(+), 202 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a09485a3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 95126c6..c198674 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -29,6 +29,7 @@ import java.util.stream.Stream;
 
 import com.google.common.base.Optional;
 import org.apache.usergrid.persistence.index.impl.IndexProducer;
+import org.apache.usergrid.persistence.queue.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -56,10 +57,6 @@ import 
org.apache.usergrid.persistence.index.IndexLocationStrategy;
 import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
 import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
 
 import com.codahale.metrics.Counter;
@@ -89,6 +86,7 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 private final QueueManager queue;
 private final QueueScope queueScope;
 private final IndexProcessorFig indexProcessorFig;
+private final QueueFig queueFig;
 private final IndexProducer indexProducer;
 private final EntityCollectionManagerFactory 
entityCollectionManagerFactory;
 private final IndexLocationStrategyFactory indexLocationStrategyFactory;
@@ -115,15 +113,16 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 
 
 @Inject
-public AmazonAsyncEventService( final QueueManagerFactory 
queueManagerFactory,
-final IndexProcessorFig indexProcessorFig,
-final IndexProducer indexProducer,
-final MetricsFactory metricsFactory,
-final EntityCollectionManagerFactory 
entityCollectionManagerFactory,
-final IndexLocationStrategyFactory 
indexLocationStrategyFactory,
-final EntityIndexFactory 
entityIndexFactory,
-final EventBuilder eventBuilder,
-final RxTaskScheduler rxTaskScheduler ) {
+public AmazonAsyncEventService(final QueueManagerFactory 
queueManagerFactory,
+   final IndexProcessorFig indexProcessorFig,
+   final IndexProducer indexProducer,
+   final

[2/2] usergrid git commit: Merge branch 'refs/heads/2.1-release' into USERGRID-1048

2015-10-19 Thread toddnine 
Merge branch 'refs/heads/2.1-release' into USERGRID-1048


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3e155852
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3e155852
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3e155852

Branch: refs/heads/USERGRID-1048
Commit: 3e1558524728c96834cfd66d9e53e3f1b6a7d3d6
Parents: 04a3f47 a09485a
Author: Todd Nine 
Authored: Mon Oct 19 13:30:04 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 19 13:30:04 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java| 149 ++-
 .../asyncevents/AsyncIndexProvider.java |  26 ++--
 .../asyncevents/model/AsyncEvent.java   |  14 +-
 .../asyncevents/model/EdgeDeleteEvent.java  |   6 +-
 .../asyncevents/model/EdgeIndexEvent.java   |   9 +-
 .../asyncevents/model/EntityDeleteEvent.java|   8 +-
 .../asyncevents/model/EntityIndexEvent.java |   6 +-
 .../model/InitializeApplicationIndexEvent.java  |   4 +-
 .../index/AmazonAsyncEventServiceTest.java  |   6 +-
 .../cache/CachedEntityCollectionManager.java| 147 --
 .../EntityCollectionManagerFactoryImpl.java |   6 -
 .../usergrid/persistence/queue/QueueFig.java|   2 +-
 .../queue/impl/SNSQueueManagerImpl.java |   8 +-
 13 files changed, 135 insertions(+), 256 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3e155852/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --cc 
stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index d319ac8,c198674..f8ef5e7
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@@ -97,7 -84,9 +100,8 @@@ public class AmazonAsyncEventService im
  public static final String QUEUE_NAME = "index"; //keep this short as AWS 
limits queue name size to 80 chars
  
  private final QueueManager queue;
 -private final QueueScope queueScope;
  private final IndexProcessorFig indexProcessorFig;
+ private final QueueFig queueFig;
  private final IndexProducer indexProducer;
  private final EntityCollectionManagerFactory 
entityCollectionManagerFactory;
  private final IndexLocationStrategyFactory indexLocationStrategyFactory;
@@@ -125,33 -113,28 +129,35 @@@
  
  
  @Inject
 -public AmazonAsyncEventService(final QueueManagerFactory 
queueManagerFactory,
 -   final IndexProcessorFig indexProcessorFig,
 -   final IndexProducer indexProducer,
 -   final MetricsFactory metricsFactory,
 -   final EntityCollectionManagerFactory 
entityCollectionManagerFactory,
 -   final IndexLocationStrategyFactory 
indexLocationStrategyFactory,
 -   final EntityIndexFactory 
entityIndexFactory,
 -   final EventBuilder eventBuilder,
 -   final RxTaskScheduler rxTaskScheduler,
 -   QueueFig queueFig) {
 +public AmazonAsyncEventService( final QueueManagerFactory 
queueManagerFactory,
 +final IndexProcessorFig indexProcessorFig,
 +final IndexProducer indexProducer,
 +final MetricsFactory metricsFactory,
 +final EntityCollectionManagerFactory 
entityCollectionManagerFactory,
 +final IndexLocationStrategyFactory 
indexLocationStrategyFactory,
 +final EntityIndexFactory 
entityIndexFactory,
 +final EventBuilder eventBuilder,
 +final MapManagerFactory mapManagerFactory,
++final QueueFig queueFig,
 +final RxTaskScheduler rxTaskScheduler ) {
  this.indexProducer = indexProducer;
  
  this.entityCollectionManagerFactory = entityCollectionManagerFactory;
  this.indexLocationStrategyFactory = indexLocationStrategyFactory;
  this.entityIndexFactory = entityIndexFactory;
  this.eventBuilder = eventBuilder;
 +
 +final MapScope mapScope = new MapScopeImpl( 
CpNamingUtils.getManagementApplicationId(),  "indexEvents");
 +
 +this.esMapPersistence =

usergrid git commit: USERGRID-1044: add optional region header functionality to most requests

2015-10-19 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 06e3fa49a -> 963a08417


USERGRID-1044: add optional region header functionality to most requests


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/963a0841
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/963a0841
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/963a0841

Branch: refs/heads/USERGRID-1044
Commit: 963a08417aa66eea567301e2592751d051a1a73c
Parents: 06e3fa4
Author: Mike Dunker 
Authored: Mon Oct 19 11:07:41 2015 -0700
Committer: Mike Dunker 
Committed: Mon Oct 19 11:07:41 2015 -0700

--
 .../org/apache/usergrid/scenarios/AuditScenarios.scala   |  3 +++
 .../usergrid/scenarios/EntityCollectionScenarios.scala   | 11 +++
 2 files changed, 14 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/963a0841/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index bd0d007..be754f4 100644
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -58,6 +58,7 @@ object AuditScenarios {
 http("GET collections")
   .get(collectionGetUrl(false))
   .headers(Headers.authToken)
+  .headers(Headers.auditRegionHeaders)
   
.check(status.is(200),extractAuditEntities(SessionVarCollectionEntities),maybeExtractCursor(SessionVarCursor)))
   .foreach("${" + SessionVarCollectionEntities + "}", "singleResult") {
 exec(session => {
@@ -75,6 +76,7 @@ object AuditScenarios {
 http("GET collections")
   .get(collectionGetUrl(true))
   .headers(Headers.authToken)
+  .headers(Headers.auditRegionHeaders)
   
.check(status.is(200),extractAuditEntities(SessionVarCollectionEntities),maybeExtractCursor(SessionVarCursor)))
   .foreach("${" + SessionVarCollectionEntities + "}", "singleResult") {
 exec(session => {
@@ -118,6 +120,7 @@ object AuditScenarios {
 http("DELETE audited entity")
   .delete("/${collectionName}/${uuid}")
   .headers(Headers.authToken)
+  .headers(Headers.auditRegionHeaders)
   .check(extractEntityUuid(SessionVarDeletedUuid)))
   .exec(session => {
 val uuid = session(SessionVarDeletedUuid).as[String]

http://git-wip-us.apache.org/repos/asf/usergrid/blob/963a0841/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index de95920..bc3cf8c 100644
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -73,6 +73,7 @@ object EntityCollectionScenarios {
 http("GET entities")
   .get(entityGetUrl(false))
   .headers(Headers.authToken)
+  .headers(Headers.auditRegionHeaders)
   .check(status.is(200), maybeExtractCursor(SessionVarCursor))
   )
 
@@ -80,6 +81,7 @@ object EntityCollectionScenarios {
 http("GET entities")
   .get(entityGetUrl(true))
   .headers(Headers.authToken)
+  .headers(Headers.auditRegionHeaders)
   .check(status.is(200), maybeExtractCursor(SessionVarCursor))
   )
 
@@ -110,6 +112,7 @@ object EntityCollectionScenarios {
 http("GET entity by name (anonymous)")
   .get(randomEntityNameUrl())
   .headers(Headers.authAnonymous)
+  .headers(Headers.auditRegionHeaders)
   .check(status.is(200))
   )
 
@@ -117,6 +120,7 @@ object EntityCollectionScenarios {
 http("GET entity by name (token)")
   .get(randomEntityNameUrl())
   .headers(Headers.authToken)
+  .headers(Headers.auditRegionHeaders)
   .check(status.is(200))
   )
 
@@ -153,6 +157,7 @@ object EntityCollectionScenarios {
   .get("/" + Settings.collection + "/${uuid}")
   .queryParamMap(Settings.queryParamMap)
   .headers(Headers.authAnonymous)
+  .headers(Headers.auditRegionHeaders)
   .check(status.is(200))
   )
 
@@ -161,6 +166,7 @@ object EntityCollectionScenarios {
   .get("/" + Settings.collection + "/${uuid}")
   .queryParamMap(Settings.queryParamMap)
   .headers(Headers.authToken)
+  .headers(Headers.auditRegionHeaders)

[1/3] usergrid git commit: Add sourceRegion to SQS event message based on queue primary region. Remove cached entity manager (unused).

2015-10-19 Thread sfeldman 
Repository: usergrid
Updated Branches:
  refs/heads/remove-inmemory-event-service efffe474b -> 00175cc61


Add sourceRegion to SQS event message based on queue primary region.  Remove 
cached entity manager (unused).


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a09485a3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a09485a3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a09485a3

Branch: refs/heads/remove-inmemory-event-service
Commit: a09485a3a5ac8b4217b294f4754ea8a70a7ec447
Parents: 2b22c61
Author: Michael Russo 
Authored: Fri Oct 16 13:12:42 2015 -0700
Committer: Michael Russo 
Committed: Fri Oct 16 13:12:42 2015 -0700

--
 .../asyncevents/AmazonAsyncEventService.java|  40 ++---
 .../asyncevents/AsyncIndexProvider.java |  10 +-
 .../asyncevents/model/AsyncEvent.java   |  14 +-
 .../asyncevents/model/EdgeDeleteEvent.java  |   6 +-
 .../asyncevents/model/EdgeIndexEvent.java   |   9 +-
 .../asyncevents/model/EntityDeleteEvent.java|   8 +-
 .../asyncevents/model/EntityIndexEvent.java |   6 +-
 .../model/InitializeApplicationIndexEvent.java  |   4 +-
 .../index/AmazonAsyncEventServiceTest.java  |   6 +-
 .../cache/CachedEntityCollectionManager.java| 147 ---
 .../EntityCollectionManagerFactoryImpl.java |   6 -
 .../usergrid/persistence/queue/QueueFig.java|   2 +-
 .../queue/impl/SNSQueueManagerImpl.java |   6 +-
 .../queue/impl/SQSQueueManagerImpl.java |   2 +-
 14 files changed, 64 insertions(+), 202 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a09485a3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 95126c6..c198674 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -29,6 +29,7 @@ import java.util.stream.Stream;
 
 import com.google.common.base.Optional;
 import org.apache.usergrid.persistence.index.impl.IndexProducer;
+import org.apache.usergrid.persistence.queue.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -56,10 +57,6 @@ import 
org.apache.usergrid.persistence.index.IndexLocationStrategy;
 import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
 import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
 
 import com.codahale.metrics.Counter;
@@ -89,6 +86,7 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 private final QueueManager queue;
 private final QueueScope queueScope;
 private final IndexProcessorFig indexProcessorFig;
+private final QueueFig queueFig;
 private final IndexProducer indexProducer;
 private final EntityCollectionManagerFactory 
entityCollectionManagerFactory;
 private final IndexLocationStrategyFactory indexLocationStrategyFactory;
@@ -115,15 +113,16 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 
 
 @Inject
-public AmazonAsyncEventService( final QueueManagerFactory 
queueManagerFactory,
-final IndexProcessorFig indexProcessorFig,
-final IndexProducer indexProducer,
-final MetricsFactory metricsFactory,
-final EntityCollectionManagerFactory 
entityCollectionManagerFactory,
-final IndexLocationStrategyFactory 
indexLocationStrategyFactory,
-final EntityIndexFactory 
entityIndexFactory,
-final EventBuilder eventBuilder,
-final RxTaskScheduler rxTaskScheduler ) {
+public AmazonAsyncEventService(final QueueManagerFactory 
queueManagerFactory,
+   final IndexProcessorFig indexProcessorFig,
+   final IndexProducer indexProducer,
+

usergrid git commit: USERGRID-1044: add region to unlimited load test plus rename "auditRegion" to "usergridRegion" since it can be used for non-audit purposes

2015-10-19 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 963a08417 -> 2f307a750


USERGRID-1044: add region to unlimited load test
plus rename "auditRegion" to "usergridRegion" since it can be used for
non-audit purposes


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2f307a75
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2f307a75
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2f307a75

Branch: refs/heads/USERGRID-1044
Commit: 2f307a75066204a894bd68115518e29064ae56cc
Parents: 963a084
Author: Mike Dunker 
Authored: Mon Oct 19 12:56:22 2015 -0700
Committer: Mike Dunker 
Committed: Mon Oct 19 12:56:22 2015 -0700

--
 .../runAuditVerifyCollectionEntities.sh |  7 +++
 stack/loadtests/runLoadEntitiesUnlimited.sh |  4 +++-
 .../usergrid/enums/ConfigProperties.scala   |  6 +++---
 .../org/apache/usergrid/helpers/Headers.scala   |  4 ++--
 .../usergrid/scenarios/AuditScenarios.scala | 10 -
 .../scenarios/EntityCollectionScenarios.scala   | 22 ++--
 .../org/apache/usergrid/settings/Settings.scala |  2 +-
 stack/loadtests/testConfig.sh   |  1 +
 8 files changed, 29 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/2f307a75/stack/loadtests/runAuditVerifyCollectionEntities.sh
--
diff --git a/stack/loadtests/runAuditVerifyCollectionEntities.sh 
b/stack/loadtests/runAuditVerifyCollectionEntities.sh
index fa75505..695c9ee 100755
--- a/stack/loadtests/runAuditVerifyCollectionEntities.sh
+++ b/stack/loadtests/runAuditVerifyCollectionEntities.sh
@@ -28,18 +28,17 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 #RETRY_COUNT=
 
 DELETE_AFTER_SUCCESSFUL_AUDIT=false
-AUDIT_REGION=
 
 die() { echo "$@" 1>&2 ; exit 1; }
 
-[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided.  Example is 
$0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME 
[DELETE_AFTER_SUCCESSFUL_AUDIT(true/false)[AUDIT_REGION]]]"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided.  Example is 
$0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME 
[DELETE_AFTER_SUCCESSFUL_AUDIT(true/false)[USERGRID_REGION]]]"
 
 RAMP_USERS="$1"
 RAMP_TIME="$2"
 AUDIT_UUID_FILENAME="$3"
 FAILED_UUID_FILENAME="$4"
 [ "$#" -ge 5 ] && DELETE_AFTER_SUCCESSFUL_AUDIT="$5"
-[ "$#" -ge 6 ] && AUDIT_REGION="$6"
+[ "$#" -ge 6 ] && USERGRID_REGION="$6"
 
 shift $#
 
@@ -65,6 +64,6 @@ mvn gatling:execute \
 -DfailedUuidFilename=${FAILED_UUID_FILENAME} \
 -DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
 -DdeleteAfterSuccessfulAudit=${DELETE_AFTER_SUCCESSFUL_AUDIT} \
--DauditRegion=${AUDIT_REGION} \
+-DusergridRegion=${USERGRID_REGION} \
 -Dgatling.simulationClass=org.apache.usergrid.simulations.AuditSimulation
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/2f307a75/stack/loadtests/runLoadEntitiesUnlimited.sh
--
diff --git a/stack/loadtests/runLoadEntitiesUnlimited.sh 
b/stack/loadtests/runLoadEntitiesUnlimited.sh
index 9222a57..11ca0a4 100755
--- a/stack/loadtests/runLoadEntitiesUnlimited.sh
+++ b/stack/loadtests/runLoadEntitiesUnlimited.sh
@@ -44,7 +44,7 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 
 die() { echo "$@" 1>&2 ; exit 1; }
 
-[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided.  Example is 
$0 RAMP_USERS RAMP_TIME(seconds) [UUID_FILENAME [ENTITY_SEED [ENTITY_WORKER_NUM 
[ENTITY_WORKER_COUNT"
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided.  Example is 
$0 RAMP_USERS RAMP_TIME(seconds) [UUID_FILENAME [ENTITY_SEED [ENTITY_WORKER_NUM 
[ENTITY_WORKER_COUNT [USERGRID_REGION]"
 
 RAMP_USERS="$1"
 RAMP_TIME="$2"
@@ -52,6 +52,7 @@ RAMP_TIME="$2"
 [ "$#" -ge 4 ] && ENTITY_SEED="$4"
 [ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5"
 [ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6"
+[ "$#" -ge 7 ] && USERGRID_REGION="$6"
 
 shift $#
 
@@ -97,5 +98,6 @@ mvn gatling:execute \
 -DrampTime=${RAMP_TIME}  \
 -DuuidFilename=${UUID_FILENAME} \
 -DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
+-DusergridRegion=${USERGRID_REGION} \
 
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/2f307a75/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index d825bb6..f74222e 100644
---

usergrid git commit: Fixes serialization tests and verifies full end to end functionality.

2015-10-19 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1048 94a907812 -> 04a3f47bd


Fixes serialization tests and verifies full end to end functionality.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/04a3f47b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/04a3f47b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/04a3f47b

Branch: refs/heads/USERGRID-1048
Commit: 04a3f47bd86ec0674ff487f479327e0f174f0425
Parents: 94a9078
Author: Todd Nine 
Authored: Mon Oct 19 12:00:56 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 19 12:00:56 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java|  70 ++-
 .../asyncevents/model/AsyncEvent.java   |   5 +-
 .../index/IndexProcessorFig.java|   7 +-
 .../util/ObjectJsonSerializer.java  |  28 +-
 .../persistence/queue/QueueManager.java |   2 +-
 .../persistence/queue/guice/QueueModule.java|   1 -
 .../queue/impl/SNSQueueManagerImpl.java | 515 ++-
 .../queue/impl/SQSQueueManagerImpl.java | 362 -
 8 files changed, 336 insertions(+), 654 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/04a3f47b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index c9f0953..d319ac8 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -29,19 +29,13 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.google.common.base.Optional;
-
-import 
org.apache.usergrid.corepersistence.asyncevents.model.ElasticsearchIndexEvent;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.corepersistence.util.ObjectJsonSerializer;
-import org.apache.usergrid.exception.NotImplementedException;
-import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.usergrid.corepersistence.asyncevents.model.AsyncEvent;
 import org.apache.usergrid.corepersistence.asyncevents.model.EdgeDeleteEvent;
 import org.apache.usergrid.corepersistence.asyncevents.model.EdgeIndexEvent;
+import 
org.apache.usergrid.corepersistence.asyncevents.model.ElasticsearchIndexEvent;
 import org.apache.usergrid.corepersistence.asyncevents.model.EntityDeleteEvent;
 import org.apache.usergrid.corepersistence.asyncevents.model.EntityIndexEvent;
 import 
org.apache.usergrid.corepersistence.asyncevents.model.InitializeApplicationIndexEvent;
@@ -50,6 +44,8 @@ import 
org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
 import org.apache.usergrid.corepersistence.index.IndexProcessorFig;
 import 
org.apache.usergrid.corepersistence.index.ReplicatedIndexLocationStrategy;
 import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.corepersistence.util.ObjectJsonSerializer;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
 import 
org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
 import 
org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
@@ -61,6 +57,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexLocationStrategy;
 import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.persistence.map.MapManagerFactory;
 import org.apache.usergrid.persistence.map.MapScope;
@@ -78,6 +75,7 @@ import com.codahale.metrics.Counter;
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Timer;
+import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
@@ -94,8 +92,6 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 
 private static final Logger logger =

usergrid git commit: Updates the defaults to be more sensible in a multi-region environment

2015-10-19 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1048 4013f17e4 -> 3ec0f5886


Updates the defaults to be more sensible in a multi-region environment


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3ec0f588
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3ec0f588
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3ec0f588

Branch: refs/heads/USERGRID-1048
Commit: 3ec0f5886b82737d4a7ed64fae01afbdb6707763
Parents: 4013f17
Author: Todd Nine 
Authored: Mon Oct 19 17:44:39 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 19 17:44:39 2015 -0600

--
 .../corepersistence/index/PublishRxTest.java|  95 
 .../usergrid/corepersistence/index/RxTest.java  | 108 +++
 .../persistence/core/astyanax/CassandraFig.java |   6 +-
 3 files changed, 111 insertions(+), 98 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3ec0f588/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
--
diff --git 
a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
 
b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
deleted file mode 100644
index 973a42d..000
--- 
a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-import rx.Observable;
-import rx.Subscription;
-import rx.observables.ConnectableObservable;
-import rx.schedulers.Schedulers;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-/**
- * Test to test some assumptions about RX behaviors
- */
-public class PublishRxTest {
-
-@Test
-public void testPublish() throws InterruptedException {
-
-final int count = 10;
-
-final CountDownLatch latch = new CountDownLatch( count );
-
-final Subscription connectedObservable =
-Observable.range( 0, count ).doOnNext( integer -> 
latch.countDown() ).subscribeOn( Schedulers.io() )
-  .subscribe();
-
-
-final boolean completed = latch.await( 5, TimeUnit.SECONDS );
-
-assertTrue( "publish1 behaves as expected", completed );
-
-final boolean completedSubscription = 
connectedObservable.isUnsubscribed();
-
-assertTrue( "Subscription complete", completedSubscription );
-}
-
-
-@Test
-@Ignore("This seems like it should work, yet blocks forever")
-public void testConnectableObserver() throws InterruptedException {
-
-final int count = 10;
-
-final CountDownLatch latch = new CountDownLatch( count );
-
-final ConnectableObservable connectedObservable = 
Observable.range( 0, count ).publish();
-
-
-//connect to our latch, which should run on it's own subscription
-//start our latch running
-connectedObservable.doOnNext( integer -> latch.countDown() 
).subscribeOn( Schedulers.io() ).subscribe();
-
-
-final Observable countObservable = 
connectedObservable.subscribeOn( Schedulers.io() ).count();
-
-//start the sequence
-connectedObservable.connect();
-
-
-final boolean completed = latch.await( 5, TimeUnit.SECONDS );
-
-assertTrue( "publish1 behaves as expected", completed );
-
-final int returnedCount = countObservable.toBlocking().last();
-
-assertEquals( "Counts the same", count, returnedCount );
-}
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3ec0f588/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
--
diff --git

usergrid git commit: Added logging changes for people who don't know how to use log4j. Added catch for duplicate usernames.

2015-10-19 Thread grey 
Repository: usergrid
Updated Branches:
  refs/heads/AdminPointerFix 6e2ff9613 -> afe0c51d6


Added logging changes for people who don't know how to use log4j.
Added catch for duplicate usernames.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/afe0c51d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/afe0c51d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/afe0c51d

Branch: refs/heads/AdminPointerFix
Commit: afe0c51d67ca1d3f262890ec44363200128e7361
Parents: 6e2ff96
Author: George Reyes 
Authored: Mon Oct 19 16:52:52 2015 -0700
Committer: George Reyes 
Committed: Mon Oct 19 16:52:52 2015 -0700

--
 .../apache/usergrid/tools/AdminPointerFix.java   | 19 ++-
 1 file changed, 18 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/afe0c51d/stack/tools/src/main/java/org/apache/usergrid/tools/AdminPointerFix.java
--
diff --git 
a/stack/tools/src/main/java/org/apache/usergrid/tools/AdminPointerFix.java 
b/stack/tools/src/main/java/org/apache/usergrid/tools/AdminPointerFix.java
index 72ae4d7..6243571 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/AdminPointerFix.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/AdminPointerFix.java
@@ -38,6 +38,7 @@ import org.apache.usergrid.persistence.Results;
 import org.apache.usergrid.persistence.cassandra.CassandraService;
 import org.apache.usergrid.persistence.entities.Application;
 import org.apache.usergrid.persistence.entities.User;
+import 
org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
 
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
@@ -78,6 +79,7 @@ public class AdminPointerFix extends ExportingToolBase {
 startSpring();
 
 logger.info( "Starting crawl of all admins" );
+System.out.println( "Starting crawl of all admins" );
 
 EntityManager em = emf.getEntityManager( 
CassandraService.MANAGEMENT_APPLICATION_ID );
 Application app = em.getApplication();
@@ -115,6 +117,8 @@ public class AdminPointerFix extends ExportingToolBase {
 
 if ( ids.size() > 1 ) {
 logger.info( "Found multiple users with the username {}", 
username );
+System.out.println("Found multiple users with the username"+ 
username+". Run DupAdmiRepair.");
+
 }
 }
 
@@ -123,6 +127,8 @@ public class AdminPointerFix extends ExportingToolBase {
 
 if ( ids.size() > 1 ) {
 logger.info( "Found multiple users with the email {}", email );
+System.out.println("Found multiple users with the username"+ 
email+". Run DupAdmiRepair.");
+
 }
 
 
@@ -136,13 +142,24 @@ public class AdminPointerFix extends ExportingToolBase {
 UUID toLoad = tempIds.get( 0 );
 
 logger.warn( "Could not load target user by email {}, loading 
by UUID {} instead", email, toLoad );
+System.out.println("Could not load the target user by email: 
"+email+". Loading by the following uuid instead: "+toLoad.toString());
 targetUser = managementService.getAdminUserByUuid( toLoad );
 User targetUserEntity = em.get( targetUser.getUuid(), 
User.class );
-em.update( targetUserEntity );
+try {
+em.update( targetUserEntity );
+}catch(DuplicateUniquePropertyExistsException dup){
+System.out.println("Found duplicate unique property: 
"+dup.getPropertyName()+ ". Duplicate property is: "+dup.getPropertyValue());
+if (dup.getPropertyName().equals( "username" )){
+   targetUserEntity.setUsername( 
targetUserEntity.getEmail() );
+}
+else
+throw dup;
+}
 
 }
 }
 
 logger.info( "Repair complete" );
+System.out.println("Repair Complete");
 }
 }

[2/2] usergrid git commit: Merge branch 'USERGRID-1048' of https://git-wip-us.apache.org/repos/asf/usergrid into USERGRID-1048

2015-10-19 Thread mrusso 
Merge branch 'USERGRID-1048' of 
https://git-wip-us.apache.org/repos/asf/usergrid into USERGRID-1048


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d8e65721
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d8e65721
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d8e65721

Branch: refs/heads/USERGRID-1048
Commit: d8e6572196b1c9245854ce1351d4d2171fbde80b
Parents: 3a7e60b 3ec0f58
Author: Michael Russo 
Authored: Mon Oct 19 17:07:31 2015 -0700
Committer: Michael Russo 
Committed: Mon Oct 19 17:07:31 2015 -0700

--
 .../asyncevents/AmazonAsyncEventService.java|   4 +
 .../corepersistence/index/PublishRxTest.java|  95 
 .../usergrid/corepersistence/index/RxTest.java  | 108 +++
 .../persistence/core/astyanax/CassandraFig.java |   6 +-
 4 files changed, 115 insertions(+), 98 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/d8e65721/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--

usergrid git commit: Add null check back for batch sending of SQS w/proper SQS client. Less logging would occur in the event of batch send when the client is not initialized.

2015-10-19 Thread mrusso 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1048 d8e657219 -> fbb6c823b


Add null check back for batch sending of SQS w/proper SQS client.  Less logging 
would occur in the event of batch send when the client is not initialized.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fbb6c823
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fbb6c823
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fbb6c823

Branch: refs/heads/USERGRID-1048
Commit: fbb6c823b7e2a1f5f55ab044942b38d1157970c0
Parents: d8e6572
Author: Michael Russo 
Authored: Mon Oct 19 21:28:46 2015 -0700
Committer: Michael Russo 
Committed: Mon Oct 19 21:28:46 2015 -0700

--
 .../usergrid/persistence/queue/impl/SNSQueueManagerImpl.java| 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbb6c823/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
--
diff --git 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index 3c18992..1bb00dc 100644
--- 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -543,6 +543,11 @@ public class SNSQueueManagerImpl implements QueueManager {
 @Override
 public void sendMessages( final List bodies ) throws IOException {
 
+if ( sqsAsync == null ) {
+logger.error( "SQS client is null, perhaps it failed to initialize 
successfully" );
+return;
+}   
+
 for ( Object body : bodies ) {
 sendMessage( ( Serializable ) body );
 }