usergrid git commit: USERGRID-1044: make jsonPath extraction optional to keep users from exiting

2015-10-27 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 1a6de02ab -> 3c60c49a1


USERGRID-1044: make jsonPath extraction optional to keep users from exiting


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3c60c49a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3c60c49a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3c60c49a

Branch: refs/heads/USERGRID-1044
Commit: 3c60c49a157be63ec626ff88666472f2d4876554
Parents: 1a6de02
Author: Mike Dunker 
Authored: Tue Oct 27 08:52:17 2015 -0700
Committer: Mike Dunker 
Committed: Tue Oct 27 08:52:17 2015 -0700

--
 .../org/apache/usergrid/helpers/Extractors.scala  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3c60c49a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala 
b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index 136a561..b21f9d7 100644
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -37,7 +37,7 @@ object Extractors {
 jsonPath("$.cursor").transformOption(extract => {
   //it may or may not be present.  If it is, save it, otherwise save it as 
an empty string
   extract.orElse(Some(""))
-}).saveAs(saveAsName)
+}).optional.saveAs(saveAsName)
   }
 
   /**
@@ -47,7 +47,7 @@ object Extractors {
 jsonPath("$.entities[0].uuid").transformOption(extract => {
   //it may or may not be present.  If it is, save it, otherwise save it as 
an empty string
   extract.orElse(Some(""))
-}).saveAs(saveAsName)
+}).optional.saveAs(saveAsName)
   }
 
   /**
@@ -57,7 +57,7 @@ object Extractors {
 jsonPath("$.entities[0].name").transformOption(extract => {
   //it may or may not be present.  If it is, save it, otherwise save it as 
an empty string
   extract.orElse(Some(""))
-}).saveAs(saveAsName)
+}).optional.saveAs(saveAsName)
   }
 
   /**
@@ -67,21 +67,25 @@ object Extractors {
 jsonPath("$.entities[0].modified").ofType[Long].transformOption(extract => 
{
   //it may or may not be present.  If it is, save it, otherwise save it as 
-1
   extract.orElse(Some(-1))
-}).saveAs(saveAsName)
+}).optional.saveAs(saveAsName)
   }
 
   /**
* Will extract the audit entities from the get collection response.
*/
   def extractAuditEntities(saveAsName: String) = {
-
jsonPath("$.entities[*]").ofType[Map[String,Any]].findAll.transformOption(extract
 => { extract.orElse(Some(Seq.empty)) }).saveAs(saveAsName)
+
jsonPath("$.entities[*]").ofType[Map[String,Any]].findAll.transformOption(extract
 => {
+  extract.orElse(Some(Seq.empty))
+}).optional.saveAs(saveAsName)
   }
 
   /**
* Will extract the audit entities from the get collection response.
*/
   def extractAuditEntity(saveAsName: String) = {
-
jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract
 => { extract.orElse(Some(Seq.empty)) }).saveAs(saveAsName)
+
jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract
 => {
+  extract.orElse(Some(Seq.empty))
+}).optional.saveAs(saveAsName)
   }
 
   /**
@@ -100,7 +104,7 @@ object Extractors {
   def maybeExtractEntities(saveAsName: String) = {
 jsonPath("$.entities").ofType[Seq[Any]].transformOption(extract => {
   extract.orElse(Some(Seq()))
-}).saveAs(saveAsName)
+}).optional.saveAs(saveAsName)
   }
 
   /**

[1/2] usergrid git commit: Moves system endpoints into the system package

2015-10-27 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1064 3b436243e -> 7a4f36ab9


http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java 
b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
new file mode 100644
index 000..c5a6dbc
--- /dev/null
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.rest.system;
+
+
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import 
org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
+import com.google.inject.Injector;
+import com.sun.jersey.api.json.JSONWithPadding;
+
+
+@Component
+@Scope( "singleton" )
+@Produces( {
+MediaType.APPLICATION_JSON,
+"application/javascript",
+"application/x-javascript",
+"text/ecmascript",
+"application/ecmascript",
+"text/jscript"
+} )
+public class MigrateResource extends AbstractContextResource {
+
+private static final Logger logger = LoggerFactory.getLogger( 
MigrateResource.class );
+
+public MigrateResource() {
+logger.info( "SystemResource initialized" );
+}
+
+@Autowired
+private Injector guiceInjector;
+
+@RequireSystemAccess
+@PUT
+@Path( "run" )
+public JSONWithPadding migrateData( @Context UriInfo ui,
+@QueryParam( "callback" ) 
@DefaultValue( "callback" ) String callback )
+throws Exception {
+
+ApiResponse response = createApiResponse();
+response.setAction( "Migrate Data" );
+//TODO make this use the task scheduler
+
+
+final Thread migrate = new Thread() {
+
+@Override
+public void run() {
+
+logger.info( "Migrating Schema" );
+
+try {
+getMigrationManager().migrate();
+}
+catch ( Exception e ) {
+logger.error( "Unable to migrate data", e );
+}
+
+logger.info( "Migrating Data" );
+
+try {
+getDataMigrationManager().migrate();
+}
+catch ( Exception e ) {
+logger.error( "Unable to migrate data", e );
+}
+}
+};
+
+migrate.setName( "Index migrate data formats" );
+migrate.setDaemon( true );
+migrate.start();
+
+response.setSuccess();
+
+return new JSONWithPadding( response, callback );
+}
+
+@RequireSystemAccess
+@PUT
+@Path( "run/{pluginName}" )
+public JSONWithPadding migrateData(@PathParam("pluginName") String 
pluginName ,  @Context UriInfo ui,
+@QueryParam( "callback" ) 
@DefaultValue( "callback" ) String callback )
+throws Exception {
+
+if(!getDataMigrationManager().pluginExists(pluginName)){
+throw new IllegalArgumentException("Plugin doesn't exits 
name:"+pluginName);
+}
+
+ApiResponse response = createApiResponse();
+response.setAction( "Migrate Data: "+ pluginName );
+

[1/5] usergrid git commit: First pass. Need to test.

2015-10-27 Thread mrusso 
Repository: usergrid
Updated Branches:
  refs/heads/2.1-release 2aec8a0fa -> e018c1e0d


First pass.  Need to test.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e41b5f02
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e41b5f02
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e41b5f02

Branch: refs/heads/2.1-release
Commit: e41b5f02eb12f72b9c19e5ae079b32d528951b47
Parents: 07d2ad3
Author: Todd Nine 
Authored: Mon Oct 26 16:24:37 2015 -0600
Committer: Todd Nine 
Committed: Mon Oct 26 16:24:37 2015 -0600

--
 .../service/StatusServiceImpl.java  |   3 +-
 .../usergrid/rest/ConnectionResource.java   | 198 +++
 .../apache/usergrid/rest/SystemResource.java|   5 +
 3 files changed, 205 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/e41b5f02/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
index 93fe653..282929e 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
@@ -93,8 +93,9 @@ public class StatusServiceImpl implements StatusService {
 final MapManager mapManager = 
mapManagerFactory.createMapManager(new MapScopeImpl(appId, "status"));
 try {
 String statusVal = mapManager.getString(jobString + statusKey);
+//nothing to emit
 if(statusVal==null){
-subscriber.onNext(null);
+subscriber.onCompleted();
 }else {
 final Map data = 
MAPPER.readValue(mapManager.getString(jobString + dataKey), Map.class);
 final Status status = Status.valueOf(statusVal);

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e41b5f02/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java 
b/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
new file mode 100644
index 000..b6a38e1
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.rest;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.json.JSONWithPadding;
+
+import rx.Observable;

usergrid git commit: USERGRID-1044: attempt #2 to preprocess response body

2015-10-27 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 2778f0a0b -> 4b2d00355


USERGRID-1044: attempt #2 to preprocess response body


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b2d0035
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b2d0035
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b2d0035

Branch: refs/heads/USERGRID-1044
Commit: 4b2d00355e5645ab5c158b4c4f8e228037879af4
Parents: 2778f0a
Author: Mike Dunker 
Authored: Tue Oct 27 11:29:16 2015 -0700
Committer: Mike Dunker 
Committed: Tue Oct 27 11:29:16 2015 -0700

--
 .../usergrid/scenarios/EntityCollectionScenarios.scala | 13 -
 1 file changed, 4 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b2d0035/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index bcdb49a..f37a2d2 100644
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -223,15 +223,10 @@ object EntityCollectionScenarios {
 .transformResponse {
   case response if response.isReceived =>
 new ResponseWrapper(response) {
-  override val body = {
-val contentType = 
response.header("content-type").getOrElse("").toLowerCase
-if (contentType.contains("json")) {
-  StringResponseBody(response.body.string, response.charset)
-} else {
-  StringResponseBody("{}", StandardCharsets.UTF_8)
-}
-  }
-  }
+  val contentType = 
response.header("content-type").getOrElse("").toLowerCase
+  val bodyStr = if (contentType.contains("json")) 
response.body.string else "[]"
+  override val body = StringResponseBody(bodyStr, response.charset)
+}
 }
 // 200 for success, 400 if already exists
 .check(status.saveAs(SessionVarStatus), 
extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))

[5/5] usergrid git commit: Merge commit 'refs/pull/414/head' of github.com:apache/usergrid into 2.1-release

2015-10-27 Thread mrusso 
Merge commit 'refs/pull/414/head' of github.com:apache/usergrid into 2.1-release


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e018c1e0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e018c1e0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e018c1e0

Branch: refs/heads/2.1-release
Commit: e018c1e0d65e3af339e36c69a4e59fe2d2a83167
Parents: 2aec8a0 7a4f36a
Author: Michael Russo 
Authored: Tue Oct 27 10:54:04 2015 -0700
Committer: Michael Russo 
Committed: Tue Oct 27 10:54:04 2015 -0700

--
 .../service/StatusServiceImpl.java  |   3 +-
 .../usergrid/rest/ApplicationsResource.java | 178 --
 .../apache/usergrid/rest/DatabaseResource.java  |  98 --
 .../org/apache/usergrid/rest/IndexResource.java | 328 --
 .../apache/usergrid/rest/MigrateResource.java   | 268 ---
 .../apache/usergrid/rest/SystemResource.java| 103 --
 .../rest/system/ApplicationsResource.java   | 178 ++
 .../rest/system/ConnectionResource.java | 202 +++
 .../usergrid/rest/system/DatabaseResource.java  | 100 ++
 .../usergrid/rest/system/IndexResource.java | 331 +++
 .../usergrid/rest/system/MigrateResource.java   | 270 +++
 .../usergrid/rest/system/SystemResource.java| 108 ++
 12 files changed, 1191 insertions(+), 976 deletions(-)
--

usergrid git commit: USERGRID-1044: transform payload to empty JSON if non-json

2015-10-27 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 3c60c49a1 -> 2778f0a0b


USERGRID-1044: transform payload to empty JSON if non-json


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2778f0a0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2778f0a0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2778f0a0

Branch: refs/heads/USERGRID-1044
Commit: 2778f0a0b196bfc09e914033392ccb49271136e0
Parents: 3c60c49
Author: Mike Dunker 
Authored: Tue Oct 27 10:49:44 2015 -0700
Committer: Mike Dunker 
Committed: Tue Oct 27 10:49:44 2015 -0700

--
 .../org/apache/usergrid/helpers/Extractors.scala   |  2 +-
 .../scenarios/EntityCollectionScenarios.scala  | 17 +
 2 files changed, 18 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/2778f0a0/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala 
b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index b21f9d7..4c6ac8d 100644
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -85,7 +85,7 @@ object Extractors {
   def extractAuditEntity(saveAsName: String) = {
 
jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract
 => {
   extract.orElse(Some(Seq.empty))
-}).optional.saveAs(saveAsName)
+}).saveAs(saveAsName)
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/2778f0a0/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index ecd0c21..bcdb49a 100644
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -16,9 +16,13 @@
  */
 package org.apache.usergrid.scenarios
 
+import java.nio.charset.StandardCharsets
+
 import io.gatling.core.Predef._
 import io.gatling.core.feeder.RecordSeqFeederBuilder
 import io.gatling.http.Predef._
+import io.gatling.http.response.ResponseWrapper
+import io.gatling.http.response.StringResponseBody
 import org.apache.usergrid.datagenerators.FeederGenerator
 import org.apache.usergrid.enums.{CsvFeedPatternType, EndConditionType, 
AuthType}
 import org.apache.usergrid.helpers.Extractors._
@@ -216,6 +220,19 @@ object EntityCollectionScenarios {
 .headers(Headers.authToken)
 .headers(Headers.usergridRegionHeaders)
 .body(StringBody("""${entity}"""))
+.transformResponse {
+  case response if response.isReceived =>
+new ResponseWrapper(response) {
+  override val body = {
+val contentType = 
response.header("content-type").getOrElse("").toLowerCase
+if (contentType.contains("json")) {
+  StringResponseBody(response.body.string, response.charset)
+} else {
+  StringResponseBody("{}", StandardCharsets.UTF_8)
+}
+  }
+  }
+}
 // 200 for success, 400 if already exists
 .check(status.saveAs(SessionVarStatus), 
extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
 .exec(session => {

[4/5] usergrid git commit: Moves system endpoints into the system package

2015-10-27 Thread mrusso 
Moves system endpoints into the system package


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7a4f36ab
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7a4f36ab
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7a4f36ab

Branch: refs/heads/2.1-release
Commit: 7a4f36ab9c0e504133f80f4d0ad8a70d115ee4f4
Parents: 3b43624
Author: Todd Nine 
Authored: Tue Oct 27 09:46:06 2015 -0600
Committer: Todd Nine 
Committed: Tue Oct 27 09:46:06 2015 -0600

--
 .../usergrid/rest/ApplicationsResource.java | 178 --
 .../usergrid/rest/ConnectionResource.java   | 199 ---
 .../apache/usergrid/rest/DatabaseResource.java  |  98 --
 .../org/apache/usergrid/rest/IndexResource.java | 328 --
 .../apache/usergrid/rest/MigrateResource.java   | 268 ---
 .../apache/usergrid/rest/SystemResource.java| 108 --
 .../rest/system/ApplicationsResource.java   | 178 ++
 .../rest/system/ConnectionResource.java | 202 +++
 .../usergrid/rest/system/DatabaseResource.java  | 100 ++
 .../usergrid/rest/system/IndexResource.java | 331 +++
 .../usergrid/rest/system/MigrateResource.java   | 270 +++
 .../usergrid/rest/system/SystemResource.java| 108 ++
 12 files changed, 1189 insertions(+), 1179 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java 
b/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
deleted file mode 100644
index 55cf0f6..000
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- *
- *  * Licensed to the Apache Software Foundation (ASF) under one or more
- *  *  contributor license agreements.  The ASF licenses this file to You
- *  * under the Apache License, Version 2.0 (the "License"); you may not
- *  * use this file except in compliance with the License.
- *  * You may obtain a copy of the License at
- *  *
- *  * http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.  For additional information regarding
- *  * copyright in this work, please see the NOTICE file in the top level
- *  * directory of this distribution.
- *
- */
-package org.apache.usergrid.rest;
-
-import com.sun.jersey.api.json.JSONWithPadding;
-import org.apache.usergrid.corepersistence.service.StatusService;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.core.util.StringUtils;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Classy class class.
- */
-@Component
-@Scope( "singleton" )
-@Produces( {
-MediaType.APPLICATION_JSON, "application/javascript", 
"application/x-javascript", "text/ecmascript",
-"application/ecmascript", "text/jscript"
-} )
-public class ApplicationsResource extends AbstractContextResource {
-
-private static final Logger logger = 
LoggerFactory.getLogger(ApplicationsResource.class);
-
-
-public ApplicationsResource() {
-
-logger.info( "ApplicationsResource initialized" );
-} {
-
-}
-
-@RequireSystemAccess
-@DELETE
-@Path( "{applicationId}" )
-public JSONWithPadding clearApplication( @Context UriInfo ui,
- @PathParam("applicationId") UUID 
applicationId,
- @QueryParam( 
"confirmApplicationName" ) String confirmApplicationName,
- @QueryParam( "limit" ) int limit,
- @QueryParam( "callback" ) 
@DefaultValue(

[3/5] usergrid git commit: Moves system endpoints into the system package

2015-10-27 Thread mrusso 
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java 
b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
new file mode 100644
index 000..c5a6dbc
--- /dev/null
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.rest.system;
+
+
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import 
org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
+import com.google.inject.Injector;
+import com.sun.jersey.api.json.JSONWithPadding;
+
+
+@Component
+@Scope( "singleton" )
+@Produces( {
+MediaType.APPLICATION_JSON,
+"application/javascript",
+"application/x-javascript",
+"text/ecmascript",
+"application/ecmascript",
+"text/jscript"
+} )
+public class MigrateResource extends AbstractContextResource {
+
+private static final Logger logger = LoggerFactory.getLogger( 
MigrateResource.class );
+
+public MigrateResource() {
+logger.info( "SystemResource initialized" );
+}
+
+@Autowired
+private Injector guiceInjector;
+
+@RequireSystemAccess
+@PUT
+@Path( "run" )
+public JSONWithPadding migrateData( @Context UriInfo ui,
+@QueryParam( "callback" ) 
@DefaultValue( "callback" ) String callback )
+throws Exception {
+
+ApiResponse response = createApiResponse();
+response.setAction( "Migrate Data" );
+//TODO make this use the task scheduler
+
+
+final Thread migrate = new Thread() {
+
+@Override
+public void run() {
+
+logger.info( "Migrating Schema" );
+
+try {
+getMigrationManager().migrate();
+}
+catch ( Exception e ) {
+logger.error( "Unable to migrate data", e );
+}
+
+logger.info( "Migrating Data" );
+
+try {
+getDataMigrationManager().migrate();
+}
+catch ( Exception e ) {
+logger.error( "Unable to migrate data", e );
+}
+}
+};
+
+migrate.setName( "Index migrate data formats" );
+migrate.setDaemon( true );
+migrate.start();
+
+response.setSuccess();
+
+return new JSONWithPadding( response, callback );
+}
+
+@RequireSystemAccess
+@PUT
+@Path( "run/{pluginName}" )
+public JSONWithPadding migrateData(@PathParam("pluginName") String 
pluginName ,  @Context UriInfo ui,
+@QueryParam( "callback" ) 
@DefaultValue( "callback" ) String callback )
+throws Exception {
+
+if(!getDataMigrationManager().pluginExists(pluginName)){
+throw new IllegalArgumentException("Plugin doesn't exits 
name:"+pluginName);
+}
+
+ApiResponse response = createApiResponse();
+response.setAction( "Migrate Data: "+ pluginName );
+//TODO make this use the task scheduler
+
+
+
+
+final Thread migrate = new Thread() {

usergrid git commit: Change read repair to interact with c* directly and only fire and index operation message to get the ES document removed from all regions.

2015-10-27 Thread mrusso 
Repository: usergrid
Updated Branches:
  refs/heads/delete-event-updates 70d7a9586 -> 1b43bda3f


Change read repair to interact with c* directly and only fire and index 
operation message to get the ES document removed from all regions.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b43bda3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b43bda3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b43bda3

Branch: refs/heads/delete-event-updates
Commit: 1b43bda3f801172b0e59f927ff3ae52a559d36cc
Parents: 70d7a95
Author: Michael Russo 
Authored: Tue Oct 27 10:50:16 2015 -0700
Committer: Michael Russo 
Committed: Tue Oct 27 12:56:13 2015 -0700

--
 .../asyncevents/AsyncEventService.java  |  5 ++
 .../asyncevents/InMemoryAsyncEventService.java  |  5 ++
 .../read/traverse/AbstractReadGraphFilter.java  | 69 +---
 .../traverse/ReadGraphCollectionFilter.java | 10 ++-
 .../traverse/ReadGraphConnectionFilter.java | 10 ++-
 .../impl/stage/NodeDeleteListenerImpl.java  | 27 
 6 files changed, 97 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index dcfffcb..dbf8996 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -77,6 +77,11 @@ public interface AsyncEventService extends ReIndexAction {
  */
 void queueEntityDelete(final ApplicationScope applicationScope, final Id 
entityId);
 
+/**
+ *
+ * @param indexOperationMessage
+ */
+void queueIndexOperationMessage( final IndexOperationMessage 
indexOperationMessage );
 
 /**
  * current queue depth

http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
index fc6385c..d8334b3 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
@@ -105,6 +105,11 @@ public class InMemoryAsyncEventService implements 
AsyncEventService {
 run( results.getCompactedNode() );
 }
 
+@Override
+public void queueIndexOperationMessage(final IndexOperationMessage 
indexOperationMessage){
+//this is not used locally
+}
+
 
 public void index( final ApplicationScope applicationScope, final Id id, 
final long updatedSince ) {
 final EntityIndexOperation entityIndexOperation = new 
EntityIndexOperation( applicationScope, id, updatedSince );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 9d050c8..89230d7 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,7 +20,11 @@
 package org.apache.usergrid.corepersistence.pipeline.read.traverse;
 
 
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
 import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,6 +46,7 @@ import com.google.common.base.Optional;

[1/2] usergrid git commit: USERGRID-1044: handle audit for CSV rows that don't have UUIDs by accessing via name

2015-10-27 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 4b2d00355 -> be0033eac


USERGRID-1044: handle audit for CSV rows that don't have UUIDs by accessing via 
name


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9158772b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9158772b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9158772b

Branch: refs/heads/USERGRID-1044
Commit: 9158772bc20befa8affa6c1bc10af032a07d0400
Parents: 4b2d003
Author: Mike Dunker 
Authored: Tue Oct 27 12:46:48 2015 -0700
Committer: Mike Dunker 
Committed: Tue Oct 27 12:46:48 2015 -0700

--
 .../org/apache/usergrid/datagenerators/FeederGenerator.scala| 5 -
 .../scala/org/apache/usergrid/scenarios/AuditScenarios.scala| 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9158772b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index b8a28d5..73b3d3c 100755
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -280,8 +280,11 @@ object FeederGenerator {
   val uuid = if (line != null) array(2) else ""
   val modified = if (line != null) array(3) else ""
   //println(s"$collectionName|$name|$uuid|$modified")
+  val accessField = if (uuid != "") uuid else name
+  val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'"
 
-  Map("collectionName" -> collectionName, "name" -> name,  "uuid" -> uuid, 
"modified" -> modified, "validEntity" -> validEntity)
+  Map("collectionName" -> collectionName, "name" -> name,  "uuid" -> uuid, 
"modified" -> modified,
+"validEntity" -> validEntity, "accessField" -> accessField, 
"queryField" -> queryField)
 }
   }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9158772b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index f90348e..7a93eb6 100644
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -144,7 +144,7 @@ object AuditScenarios {
 
   val getCollectionEntityDirect = exec(
 http("GET collection entity direct")
-  .get("/${collectionName}/${uuid}")
+  .get("/${collectionName}/${accessField}")
   .headers(Headers.authToken)
   .headers(Headers.usergridRegionHeaders)
   .check()
@@ -213,7 +213,7 @@ object AuditScenarios {
 
   val getCollectionEntity = exec(
 http("GET collection entity")
-  .get("/${collectionName}?ql=uuid=${uuid}")
+  .get("/${collectionName}?ql=${queryField}")
   .headers(Headers.authToken)
   .headers(Headers.usergridRegionHeaders)
   .check(status.is(200), status.saveAs(SessionVarStatus), 
jsonPath("$.count").optional.saveAs("count"),

usergrid git commit: Updates the message flow to allow for multiple processor threads per SQS take thread

2015-10-27 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/delete-event-updates 1b43bda3f -> 76476f17c


Updates the message flow to allow for multiple processor threads per SQS take 
thread


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/76476f17
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/76476f17
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/76476f17

Branch: refs/heads/delete-event-updates
Commit: 76476f17cf8e8be6f01660db3d21110eda8247f5
Parents: 1b43bda
Author: Todd Nine 
Authored: Tue Oct 27 14:35:34 2015 -0600
Committer: Todd Nine 
Committed: Tue Oct 27 14:35:34 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java| 68 +++-
 .../index/IndexProcessorFig.java| 13 +++-
 2 files changed, 51 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/76476f17/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index d93e304..6b9abbc 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -679,35 +679,45 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 }
 while ( true );
 }
-} )
-//this won't block our read loop, just reads and 
proceeds
-.map( messages -> {
-if ( messages == null || messages.size() == 0 
) {
-return null;
-}
-
-try {
-List indexEventResults = 
callEventHandlers( messages );
-List messagesToAck = 
submitToIndex( indexEventResults );
-if ( messagesToAck == null || 
messagesToAck.size() == 0 ) {
-logger.error( "No messages came back 
from the queue operation should have seen "
-+ messages.size(), messages );
-return messagesToAck;
-}
-if ( messagesToAck.size() < 
messages.size() ) {
-logger.error( "Missing messages from 
queue post operation", messages,
-messagesToAck );
-}
-//ack each message, but only if we didn't 
error.
-ack( messagesToAck );
-return messagesToAck;
-}
-catch ( Exception e ) {
-logger.error( "failed to ack messages to 
sqs", e );
-return null;
-//do not rethrow so we can process all of 
them
-}
-} );
+} )//this won't block our read loop, just reads 
and proceeds
+.flatMap( sqsMessages -> {
+
+//do this on a different schedule, and introduce 
concurrency with flatmap for faster processing
+return Observable.just( sqsMessages )
+
+ .map( messages -> {
+ if ( messages == null || 
messages.size() == 0 ) {
+ return null;
+ }
+
+ try {
+ List 
indexEventResults =
+ callEventHandlers( 
messages );
+ List 
messagesToAck =
+ submitToIndex( 
indexEventResults );
+ if ( messagesToAck == 
null || messagesToAck.size() == 0 ) {
+

[2/4] usergrid git commit: Change read repair to interact with c* directly and only fire and index operation message to get the ES document removed from all regions.

2015-10-27 Thread toddnine 
Change read repair to interact with c* directly and only fire and index 
operation message to get the ES document removed from all regions.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b43bda3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b43bda3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b43bda3

Branch: refs/heads/2.1-release
Commit: 1b43bda3f801172b0e59f927ff3ae52a559d36cc
Parents: 70d7a95
Author: Michael Russo 
Authored: Tue Oct 27 10:50:16 2015 -0700
Committer: Michael Russo 
Committed: Tue Oct 27 12:56:13 2015 -0700

--
 .../asyncevents/AsyncEventService.java  |  5 ++
 .../asyncevents/InMemoryAsyncEventService.java  |  5 ++
 .../read/traverse/AbstractReadGraphFilter.java  | 69 +---
 .../traverse/ReadGraphCollectionFilter.java | 10 ++-
 .../traverse/ReadGraphConnectionFilter.java | 10 ++-
 .../impl/stage/NodeDeleteListenerImpl.java  | 27 
 6 files changed, 97 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index dcfffcb..dbf8996 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -77,6 +77,11 @@ public interface AsyncEventService extends ReIndexAction {
  */
 void queueEntityDelete(final ApplicationScope applicationScope, final Id 
entityId);
 
+/**
+ *
+ * @param indexOperationMessage
+ */
+void queueIndexOperationMessage( final IndexOperationMessage 
indexOperationMessage );
 
 /**
  * current queue depth

http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
index fc6385c..d8334b3 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
@@ -105,6 +105,11 @@ public class InMemoryAsyncEventService implements 
AsyncEventService {
 run( results.getCompactedNode() );
 }
 
+@Override
+public void queueIndexOperationMessage(final IndexOperationMessage 
indexOperationMessage){
+//this is not used locally
+}
+
 
 public void index( final ApplicationScope applicationScope, final Id id, 
final long updatedSince ) {
 final EntityIndexOperation entityIndexOperation = new 
EntityIndexOperation( applicationScope, id, updatedSince );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 9d050c8..89230d7 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,7 +20,11 @@
 package org.apache.usergrid.corepersistence.pipeline.read.traverse;
 
 
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
 import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,6 +46,7 @@ import com.google.common.base.Optional;
 import rx.Observable;
 
 
+
 /**
  * Command for reading graph edges
  */
@@ -51,15 +56,21 @@ public abstract

[4/4] usergrid git commit: Merge branch 'delete-event-updates' into 2.1-release

2015-10-27 Thread toddnine 
Merge branch 'delete-event-updates' into 2.1-release


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/26860545
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/26860545
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/26860545

Branch: refs/heads/2.1-release
Commit: 26860545ece19361569370bc63496a01bb48738d
Parents: e018c1e 76476f1
Author: Todd Nine 
Authored: Tue Oct 27 14:40:23 2015 -0600
Committer: Todd Nine 
Committed: Tue Oct 27 14:40:23 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java| 68 ++
 .../asyncevents/AsyncEventService.java  |  5 ++
 .../asyncevents/InMemoryAsyncEventService.java  |  5 ++
 .../index/IndexProcessorFig.java| 13 +++-
 .../read/traverse/AbstractReadGraphFilter.java  | 72 +---
 .../traverse/ReadGraphCollectionFilter.java | 10 ++-
 .../traverse/ReadGraphConnectionFilter.java | 10 ++-
 .../impl/stage/NodeDeleteListenerImpl.java  | 27 +++-
 8 files changed, 150 insertions(+), 60 deletions(-)
--

[3/4] usergrid git commit: Updates the message flow to allow for multiple processor threads per SQS take thread

2015-10-27 Thread toddnine 
Updates the message flow to allow for multiple processor threads per SQS take 
thread


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/76476f17
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/76476f17
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/76476f17

Branch: refs/heads/2.1-release
Commit: 76476f17cf8e8be6f01660db3d21110eda8247f5
Parents: 1b43bda
Author: Todd Nine 
Authored: Tue Oct 27 14:35:34 2015 -0600
Committer: Todd Nine 
Committed: Tue Oct 27 14:35:34 2015 -0600

--
 .../asyncevents/AmazonAsyncEventService.java| 68 +++-
 .../index/IndexProcessorFig.java| 13 +++-
 2 files changed, 51 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/76476f17/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index d93e304..6b9abbc 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -679,35 +679,45 @@ public class AmazonAsyncEventService implements 
AsyncEventService {
 }
 while ( true );
 }
-} )
-//this won't block our read loop, just reads and 
proceeds
-.map( messages -> {
-if ( messages == null || messages.size() == 0 
) {
-return null;
-}
-
-try {
-List indexEventResults = 
callEventHandlers( messages );
-List messagesToAck = 
submitToIndex( indexEventResults );
-if ( messagesToAck == null || 
messagesToAck.size() == 0 ) {
-logger.error( "No messages came back 
from the queue operation should have seen "
-+ messages.size(), messages );
-return messagesToAck;
-}
-if ( messagesToAck.size() < 
messages.size() ) {
-logger.error( "Missing messages from 
queue post operation", messages,
-messagesToAck );
-}
-//ack each message, but only if we didn't 
error.
-ack( messagesToAck );
-return messagesToAck;
-}
-catch ( Exception e ) {
-logger.error( "failed to ack messages to 
sqs", e );
-return null;
-//do not rethrow so we can process all of 
them
-}
-} );
+} )//this won't block our read loop, just reads 
and proceeds
+.flatMap( sqsMessages -> {
+
+//do this on a different schedule, and introduce 
concurrency with flatmap for faster processing
+return Observable.just( sqsMessages )
+
+ .map( messages -> {
+ if ( messages == null || 
messages.size() == 0 ) {
+ return null;
+ }
+
+ try {
+ List 
indexEventResults =
+ callEventHandlers( 
messages );
+ List 
messagesToAck =
+ submitToIndex( 
indexEventResults );
+ if ( messagesToAck == 
null || messagesToAck.size() == 0 ) {
+ logger.error(
+ "No messages came 
back from

Git Push Summary

2015-10-27 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1064 [deleted] 7a4f36ab9

usergrid git commit: USERGRID-1044: print non-200s during load (including body)

2015-10-27 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 83d7a6f83 -> 038974475


USERGRID-1044: print non-200s during load (including body)


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/03897447
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/03897447
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/03897447

Branch: refs/heads/USERGRID-1044
Commit: 0389744752f479d215f84ebbd5f25aa36d2108a4
Parents: 83d7a6f
Author: Mike Dunker 
Authored: Tue Oct 27 14:59:10 2015 -0700
Committer: Mike Dunker 
Committed: Tue Oct 27 14:59:10 2015 -0700

--
 .../usergrid/scenarios/EntityCollectionScenarios.scala   | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/03897447/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index f37a2d2..d5b3101 100644
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -42,6 +42,7 @@ object EntityCollectionScenarios {
   val SessionVarStatus: String = "status"
   val SessionVarUuid: String = "createUuid"
   val SessionVarModified: String = "createModified"
+  val SessionVarBodyString: String = "bodyString"
 
   def entityGetUrl(useCursor: Boolean): String = {
 val url = s"/${Settings.collection}?" +
@@ -225,11 +226,12 @@ object EntityCollectionScenarios {
 new ResponseWrapper(response) {
   val contentType = 
response.header("content-type").getOrElse("").toLowerCase
   val bodyStr = if (contentType.contains("json")) 
response.body.string else "[]"
+  if (bodyStr == "[]") { println(">>> USING EMPTY BODY") }
   override val body = StringResponseBody(bodyStr, response.charset)
 }
 }
 // 200 for success, 400 if already exists
-.check(status.saveAs(SessionVarStatus), 
extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
+.check(status.saveAs(SessionVarStatus), 
bodyString.saveAs(SessionVarBodyString), extractEntityUuid(SessionVarUuid), 
extractEntityModified(SessionVarModified)))
 .exec(session => {
   val saveFailures = Settings.saveInvalidResponse
   val status = session(SessionVarStatus).as[Int]
@@ -239,6 +241,10 @@ object EntityCollectionScenarios {
 val entityName = session("entityName").as[String]
 val modified = if (status == 200) 
session(SessionVarModified).as[Long] else 0
 val collectionName = session("collectionName").as[String]
+if (status != 200) {
+  val bodyString = session(SessionVarBodyString).as[String]
+  println(s" LOAD ERROR - Status: 
$status\nBody:\n$bodyString")
+}
 Settings.addUuid(uuid, collectionName, entityName, modified, 
status)
 session
   } else {

usergrid git commit: USERGRID-1044: for org/app/collection create, default to testConfig settings

2015-10-27 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 038974475 -> fc23b79e9


USERGRID-1044: for org/app/collection create, default to testConfig settings


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fc23b79e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fc23b79e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fc23b79e

Branch: refs/heads/USERGRID-1044
Commit: fc23b79e9a8a08bc57d25e3e60a9718304f8f1c0
Parents: 0389744
Author: Mike Dunker 
Authored: Tue Oct 27 16:39:33 2015 -0700
Committer: Mike Dunker 
Committed: Tue Oct 27 16:39:33 2015 -0700

--
 stack/loadtests/runOrgAppSetup.sh | 17 ++---
 1 file changed, 10 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/fc23b79e/stack/loadtests/runOrgAppSetup.sh
--
diff --git a/stack/loadtests/runOrgAppSetup.sh 
b/stack/loadtests/runOrgAppSetup.sh
index 19936b7..b26eb37 100755
--- a/stack/loadtests/runOrgAppSetup.sh
+++ b/stack/loadtests/runOrgAppSetup.sh
@@ -29,6 +29,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 #ENTITY_TYPE=
 #ENTITY_PREFIX=
 #ENTITY_SEED=  #may be overridden on command line
+#ORG= #may be overridden on command line
+#APP= #may be overridden on command line
+#COLLECTION= #may be overridden on command line
 #RETRY_COUNT=
 #ENTITY_PROGRESS_COUNT=
 #CONSTANT_USERS_PER_SEC=
@@ -36,14 +39,14 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 
 die() { echo "$@" 1>&2 ; exit 1; }
 
-[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided.  Example is 
$0 ORG APP [COLLECTION [SANDBOX_COLLECTION (true/false)]]"
+[ "$#" -ge 1 ] || die "At least 1 argument required, $# provided.  Example is 
$0 SANDBOX_COLLECTION(true/false) [ORG [APP [COLLECTION]]]"
 
-ORG="$1"
-APP="$2"
-COLLECTION="gatlingitems"
-[ "$#" -ge 3 ] && COLLECTION="$3"
-SANDBOX_COLLECTION=true
-[ "$#" -ge 4 ] && SANDBOX_COLLECTION="$4"
+
+SANDBOX_COLLECTION="$1"
+# org, app, and collection can come from testConfig.sh
+[ "$#" -ge 2 ] && ORG="$2"
+[ "$#" -ge 3 ] && APP="$3"
+[ "$#" -ge 4 ] && COLLECTION="$4"
 
 shift $#

usergrid git commit: USERGRID-1044: add shutdown handler to audit simulations

2015-10-27 Thread mdunker 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1044 fc23b79e9 -> 2e70a72b9


USERGRID-1044: add shutdown handler to audit simulations


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2e70a72b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2e70a72b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2e70a72b

Branch: refs/heads/USERGRID-1044
Commit: 2e70a72b99ac0a48e5f703a1b464fdad3c4d2665
Parents: fc23b79
Author: Mike Dunker 
Authored: Tue Oct 27 16:45:02 2015 -0700
Committer: Mike Dunker 
Committed: Tue Oct 27 16:45:02 2015 -0700

--
 .../apache/usergrid/simulations/AuditSimulation.scala| 11 +++
 .../usergrid/simulations/ConfigurableSimulation.scala|  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e70a72b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
index 47323ea..bdec3dd 100755
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
@@ -57,6 +57,10 @@ class AuditSimulation extends Simulation {
   }
 
   after {
+endHandler
+  }
+
+  def endHandler: Unit = {
 Settings.setTestEndTime()
 if (Settings.captureAuditUuids) {
   val uuidDesc = Settings.scenarioType match {
@@ -69,5 +73,12 @@ class AuditSimulation extends Simulation {
 Settings.printAuditResults()
   }
 
+  def abortedEarly: Unit = {
+println("AUDIT ABORTED")
+endHandler
+  }
+
+  sys addShutdownHook abortedEarly
+
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e70a72b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
--
diff --git 
a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
 
b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
index 9d4b05d..a3eb595 100755
--- 
a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
+++ 
b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
@@ -99,7 +99,7 @@ class ConfigurableSimulation extends Simulation {
 Settings.printSettingsSummary(true)
   }
 
-  sys addShutdownHook(endHandler)
+  sys addShutdownHook endHandler
 
 }